From: heister Date: Sat, 23 Oct 2010 13:41:09 +0000 (+0000) Subject: merge branch distributed_grids up to rev 22429. X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e86f026bef0f640d367e9600316b42deedd87146;p=dealii-svn.git merge branch distributed_grids up to rev 22429. git-svn-id: https://svn.dealii.org/trunk@22430 0785d39b-7218-0410-832d-ea1e28bc413d --- diff --git a/deal.II/Makefile b/deal.II/Makefile index 7b33252f3b..b971d6cbd7 100644 --- a/deal.II/Makefile +++ b/deal.II/Makefile @@ -267,6 +267,7 @@ build-test-do: .PHONY: base baseg lac lacg 1d 2d 3d 1dg 2dg 3dg all \ online-doc doc printable-doc tex-doc contrib \ + contrib-functionparser \ clean clean-contrib clean-base clean-lac clean-dealII \ clean-doc clean-examples clean-lib clean-tests clean-common-scripts \ distclean \ diff --git a/deal.II/aclocal.m4 b/deal.II/aclocal.m4 index eb700d20a8..9256e092d2 100644 --- a/deal.II/aclocal.m4 +++ b/deal.II/aclocal.m4 @@ -319,7 +319,7 @@ AC_DEFUN(DEAL_II_DETERMINE_CXX_BRAND, dnl dnl ------------------------------------------------------------- -dnl See whether the compiler we have has MPI build in (e.g. if it +dnl See whether the compiler we have has MPI built in (e.g. if it dnl is actually mpiCC, etc) dnl dnl Usage: DEAL_II_DETERMINE_IF_SUPPORTS_MPI @@ -339,6 +339,12 @@ AC_DEFUN(DEAL_II_DETERMINE_IF_SUPPORTS_MPI, dnl AC_MSG_RESULT(yes) AC_DEFINE(DEAL_II_COMPILER_SUPPORTS_MPI, 1, [Defined if the compiler supports including ]) + + dnl Export this variable so that we can refer to it + dnl from contrib/configure.in when configuring p4est + DEAL_II_COMPILER_SUPPORTS_MPI=1 + export DEAL_II_COMPILER_SUPPORTS_MPI + DEAL_II_USE_MPI=yes ], [ @@ -1289,7 +1295,8 @@ dnl Set C compiler flags to their default values. They will be dnl modified according to other options in later steps of dnl configuration dnl -dnl CFLAGS : flags for optimized mode +dnl CFLAGS.o : flags for optimized mode +dnl CFLAGS.g : flags for debug mode dnl dnl Usage: DEAL_II_SET_CC_FLAGS dnl @@ -1298,7 +1305,8 @@ AC_DEFUN(DEAL_II_SET_CC_FLAGS, dnl [ dnl First the flags for gcc compilers if test "$GCC" = yes ; then - CFLAGS="$CFLAGS -O3 -funroll-loops -funroll-all-loops -fstrict-aliasing" + CFLAGSO="$CFLAGS -O3 -funroll-loops -funroll-all-loops -fstrict-aliasing" + CFLAGSG="$CFLAGS -g" dnl Set PIC flags. On some systems, -fpic/PIC is implied, so don't set dnl anything to avoid a warning. on AIX make sure we always pass -lpthread dnl because this seems to be somehow required to make things work. Likewise @@ -1340,18 +1348,18 @@ AC_DEFUN(DEAL_II_SET_CC_FLAGS, dnl case "$CC_VERSION" in ibm_xlc) - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGS -O2" CFLAGSPIC="-fPIC" SHLIBLD="$CXX" ;; MIPSpro*) - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGS -O2" CFLAGSPIC="-KPIC" ;; intel_icc*) - CFLAGS="$CFLAGS -O2 -unroll" + CFLAGSO="$CFLAGS -O2 -unroll" case "$CC_VERSION" in intel_icc5 | intel_icc6 | intel_icc7 | intel_icc8 | intel_icc9) CFLAGSPIC="-KPIC" @@ -1379,7 +1387,7 @@ AC_DEFUN(DEAL_II_SET_CC_FLAGS, dnl ;; esac - CFLAGS="$CFLAGS -ansi_alias -vec_report0" + CFLAGSO="$CFLAGSO -ansi_alias -vec_report0" dnl If we are on an x86 platform, add -tpp6 to optimization dnl flags @@ -1412,7 +1420,7 @@ AC_DEFUN(DEAL_II_SET_CC_FLAGS, dnl *) AC_MSG_RESULT(Unknown C compiler - using generic options) - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGSO -O2" ;; esac fi @@ -5492,6 +5500,7 @@ AC_DEFUN(DEAL_II_CONFIGURE_NETCDF, dnl ]) + dnl ------------------------------------------------------------ dnl Check whether PETSc is installed, and if so store the dnl respective links @@ -7021,3 +7030,46 @@ dnl fi fi ]) + + + +dnl ------------------------------------------------------------ +dnl Check whether P4EST is to be used to parallelize meshes +dnl +dnl Usage: DEAL_II_CONFIGURE_P4EST +dnl +dnl ------------------------------------------------------------ +AC_DEFUN(DEAL_II_CONFIGURE_P4EST, dnl +[ + AC_MSG_CHECKING(whether p4est shall be used) + + AC_ARG_WITH(p4est, + [ --with-p4est=/path/to/p4est makes deal.II use p4est to distribute meshes + on a cluster computer], + use_p4est=$withval, + use_p4est=no) + + if test "x$use_p4est" != "xno" ; then + AC_MSG_RESULT(yes) + + if test ! -d "${use_p4est}/DEBUG" -o ! -d "${use_p4est}/FAST" ; then + echo "${use_p4est}/DEBUG" + AC_MSG_ERROR([p4est directories $use_p4est/DEBUG or $use_p4est/FAST not found]) + fi + + AC_DEFINE(DEAL_II_USE_P4EST, 1, + [Defined if we are to use the p4est library to distribute + meshes on a cluster computer.]) + USE_CONTRIB_P4EST=yes + export USE_CONTRIB_P4EST + + DEAL_II_P4EST_DIR=${use_p4est} + export DEAL_II_P4EST_DIR + + CXXFLAGSG="$CXXFLAGSG -I$use_p4est/DEBUG/include" + CXXFLAGSO="$CXXFLAGSO -I$use_p4est/FAST/include" + else + AC_MSG_RESULT(no) + fi +]) + diff --git a/deal.II/base/Makefile b/deal.II/base/Makefile index 9f09356312..521d77063a 100644 --- a/deal.II/base/Makefile +++ b/deal.II/base/Makefile @@ -21,10 +21,11 @@ h-files = $(sort $(shell echo include/base/*.h)) # libbase. do similarly for the threading building block things if threading # is enabled ifeq ($(enable-parser),yes) - extra-o-files = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT) + extra-o-files = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT) extra-g.o-files = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT) endif + # production rules $(LIBDIR)/base/%.g.$(OBJEXT) : @echo "=====base=============debug======$(MT)== $((-2); } diff --git a/deal.II/base/include/base/utilities.h b/deal.II/base/include/base/utilities.h index a045cc9cec..72ece1e8a7 100644 --- a/deal.II/base/include/base/utilities.h +++ b/deal.II/base/include/base/utilities.h @@ -216,6 +216,29 @@ namespace Utilities */ double get_cpu_load (); + /** + * Structure that hold information about + * memory usage in kB. Used by + * get_memory_stats(). See man 5 proc + * entry /status for details. + */ + struct MemoryStats + { + unsigned long int VmPeak; /** peak virtual memory size in kB */ + unsigned long int VmSize; /** current virtual memory size in kB */ + unsigned long int VmHWM; /** peak resident memory size in kB */ + unsigned long int VmRSS; /** current resident memory size in kB */ + }; + + + /** + * Fills the @param stats structure with + * information about the memory + * consumption of this process. This is + * only implemented on Linux. + */ + void get_memory_stats (MemoryStats & stats); + /** * Return the name of the host this @@ -331,6 +354,38 @@ namespace Utilities */ MPI_Comm duplicate_communicator (const MPI_Comm &mpi_communicator); + + /** + * Data structure to store the result of + * calculate_collective_mpi_min_max_avg. + */ + struct MinMaxAvg + { + double sum; + double min; + double max; + unsigned int min_index; + unsigned int max_index; + double avg; + }; + + /** + * Returns sum, average, minimum, + * maximum, processor id of minimum and + * maximum as a collective operation of + * on the given MPI communicator @param + * mpi_communicator . Each processor's + * value is given in @param my_value and + * the result will be returned in @param + * result . The result is available on all + * machines. + */ + void calculate_collective_mpi_min_max_avg(const MPI_Comm &mpi_communicator, + const double my_value, + MinMaxAvg & result); + + + /** * A class that is used to initialize the * MPI system at the beginning of a diff --git a/deal.II/base/source/index_set.cc b/deal.II/base/source/index_set.cc index 3d1d4f8464..2ecf4f2bc6 100644 --- a/deal.II/base/source/index_set.cc +++ b/deal.II/base/source/index_set.cc @@ -232,6 +232,40 @@ IndexSet::read(std::istream & in) } +void +IndexSet::block_write(std::ostream & out) const +{ + Assert (out, ExcIO()); + out.write(reinterpret_cast(&index_space_size), + sizeof(index_space_size)); + size_t n_ranges = ranges.size(); + out.write(reinterpret_cast(&n_ranges), + sizeof(n_ranges)); + if (ranges.size()) + out.write (reinterpret_cast(&*ranges.begin()), + reinterpret_cast(&*ranges.end()) + - reinterpret_cast(&*ranges.begin())); + Assert (out, ExcIO()); +} + +void +IndexSet::block_read(std::istream & in) +{ + unsigned int size; + size_t n_ranges; + in.read(reinterpret_cast(&size), sizeof(size)); + in.read(reinterpret_cast(&n_ranges), sizeof(n_ranges)); + // we have to clear ranges first + ranges.clear(); + set_size(size); + ranges.resize(n_ranges, Range(0,0)); + if (n_ranges) + in.read(reinterpret_cast(&*ranges.begin()), + reinterpret_cast(&*ranges.end()) + - reinterpret_cast(&*ranges.begin())); +} + + void IndexSet::subtract_set (const IndexSet & other) diff --git a/deal.II/base/source/timer.cc b/deal.II/base/source/timer.cc index a41ce678f6..a047f14f23 100644 --- a/deal.II/base/source/timer.cc +++ b/deal.II/base/source/timer.cc @@ -96,33 +96,6 @@ void Timer::start () } -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - -void Timer::TimeMinMaxAvg::max_reduce ( const void * in_lhs_, - void * inout_rhs_, - int * len, - MPI_Datatype * ) -{ - const Timer::TimeMinMaxAvg * in_lhs = static_cast(in_lhs_); - Timer::TimeMinMaxAvg * inout_rhs = static_cast(inout_rhs_); - - Assert(*len==1, ExcInternalError()); - - inout_rhs->sum += in_lhs->sum; - if (inout_rhs->min>in_lhs->min) - { - inout_rhs->min = in_lhs->min; - inout_rhs->min_index = in_lhs->min_index; - } - if (inout_rhs->maxmax) - { - inout_rhs->max = in_lhs->max; - inout_rhs->max_index = in_lhs->max_index; - } -} - - -#endif double Timer::stop () { @@ -148,36 +121,9 @@ double Timer::stop () #ifdef DEAL_II_COMPILER_SUPPORTS_MPI if (sync_wall_time) { - unsigned int my_id = dealii::Utilities::System::get_this_mpi_process(mpi_communicator); - - MPI_Op op; - int ierr = MPI_Op_create((MPI_User_function *)&Timer::TimeMinMaxAvg::max_reduce, - false, &op); - AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); - - TimeMinMaxAvg in; - in.set(time, my_id); - - MPI_Datatype type; - int lengths[]={3,2}; - MPI_Aint displacements[]={0,offsetof(TimeMinMaxAvg, min_index)}; - MPI_Datatype types[]={MPI_DOUBLE, MPI_INT}; - - ierr = MPI_Type_struct(2, lengths, displacements, types, &type); - AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); - - ierr = MPI_Type_commit(&type); - - ierr = MPI_Reduce ( &in, &this->mpi_data, 1, type, op, 0, mpi_communicator ); - AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); - - ierr = MPI_Type_free (&type); - AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); - - ierr = MPI_Op_free(&op); - AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); - - this->mpi_data.avg = this->mpi_data.sum / dealii::Utilities::System::get_n_mpi_processes(mpi_communicator); + Utilities::System + ::calculate_collective_mpi_min_max_avg(mpi_communicator, time, + this->mpi_data); cumulative_wall_time += this->mpi_data.max; } diff --git a/deal.II/base/source/utilities.cc b/deal.II/base/source/utilities.cc index ab8e94d807..614d399807 100644 --- a/deal.II/base/source/utilities.cc +++ b/deal.II/base/source/utilities.cc @@ -423,6 +423,40 @@ namespace Utilities #endif + + void get_memory_stats (MemoryStats & stats) + { + stats.VmPeak = stats.VmSize = stats.VmHWM = stats.VmRSS = 0; + + // parsing /proc/self/stat would be a + // lot easier, but it does not contain + // VmHWM, so we use /status instead. +#if defined(__linux__) + std::ifstream file("/proc/self/status"); + std::string line; + std::string name; + while (!file.eof()) + { + file >> name; + if (name == "VmPeak:") + file >> stats.VmPeak; + else if (name == "VmSize:") + file >> stats.VmSize; + else if (name == "VmHWM:") + file >> stats.VmHWM; + else if (name == "VmRSS:") + { + file >> stats.VmRSS; + break; //this is always the last entry + } + + getline(file, line); + } +#endif + } + + + std::string get_hostname () { const unsigned int N=1024; @@ -516,6 +550,82 @@ namespace Utilities } + namespace + { + // custom MIP_Op for + // calculate_collective_mpi_min_max_avg + void max_reduce ( const void * in_lhs_, + void * inout_rhs_, + int * len, + MPI_Datatype * ) + { + const MinMaxAvg * in_lhs = static_cast(in_lhs_); + MinMaxAvg * inout_rhs = static_cast(inout_rhs_); + + Assert(*len==1, ExcInternalError()); + + inout_rhs->sum += in_lhs->sum; + if (inout_rhs->min>in_lhs->min) + { + inout_rhs->min = in_lhs->min; + inout_rhs->min_index = in_lhs->min_index; + } + else if (inout_rhs->min == in_lhs->min) + { // choose lower cpu index when tied to make operator cumutative + if (inout_rhs->min_index > in_lhs->min_index) + inout_rhs->min_index = in_lhs->min_index; + } + + if (inout_rhs->max < in_lhs->max) + { + inout_rhs->max = in_lhs->max; + inout_rhs->max_index = in_lhs->max_index; + } + else if (inout_rhs->max == in_lhs->max) + { // choose lower cpu index when tied to make operator cumutative + if (inout_rhs->max_index > in_lhs->max_index) + inout_rhs->max_index = in_lhs->max_index; + } + } + } + + void calculate_collective_mpi_min_max_avg(const MPI_Comm &mpi_communicator, + double my_value, + MinMaxAvg & result) + { + unsigned int my_id = dealii::Utilities::System::get_this_mpi_process(mpi_communicator); + unsigned int numproc = dealii::Utilities::System::get_n_mpi_processes(mpi_communicator); + + MPI_Op op; + int ierr = MPI_Op_create((MPI_User_function *)&max_reduce, true, &op); + AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); + + MinMaxAvg in; + in.sum = in.min = in.max = my_value; + in.min_index = in.max_index = my_id; + + MPI_Datatype type; + int lengths[]={3,2}; + MPI_Aint displacements[]={0,offsetof(MinMaxAvg, min_index)}; + MPI_Datatype types[]={MPI_DOUBLE, MPI_INT}; + + ierr = MPI_Type_struct(2, lengths, displacements, types, &type); + AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); + + ierr = MPI_Type_commit(&type); + ierr = MPI_Allreduce ( &in, &result, 1, type, op, mpi_communicator ); + AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); + + ierr = MPI_Type_free (&type); + AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); + + ierr = MPI_Op_free(&op); + AssertThrow(ierr == MPI_SUCCESS, ExcInternalError()); + + result.avg = result.sum / numproc; + } + + std::vector compute_point_to_point_communication_pattern (const MPI_Comm & mpi_comm, const std::vector & destinations) @@ -605,7 +715,18 @@ namespace Utilities return 0; } - + void calculate_collective_mpi_min_max_avg(const MPI_Comm &mpi_communicator, + double my_value, + MinMaxAvg & result) + { + result.sum = my_value; + result.avg = my_value; + result.min = my_value; + result.max = my_value; + result.min_index = 0; + result.max_index = 0; + } + MPI_Comm duplicate_communicator (const MPI_Comm &mpi_communicator) { diff --git a/deal.II/common/Make.global_options.in b/deal.II/common/Make.global_options.in index 5f2bc63966..01bad74bfa 100644 --- a/deal.II/common/Make.global_options.in +++ b/deal.II/common/Make.global_options.in @@ -89,6 +89,9 @@ DEAL_II_METIS_LIBDIR = @DEAL_II_METIS_LIBDIR@ USE_CONTRIB_HSL = @USE_CONTRIB_HSL@ USE_CONTRIB_UMFPACK = @USE_CONTRIB_UMFPACK@ +USE_CONTRIB_P4EST = @USE_CONTRIB_P4EST@ +DEAL_II_P4EST_DIR = @DEAL_II_P4EST_DIR@ + TARGET = @target@ ###################################################### @@ -348,7 +351,8 @@ F77FLAGS.g = @DEFS@ @F77FLAGSG@ $(INCLUDE) F77FLAGS.o = @DEFS@ @F77FLAGSO@ $(INCLUDE) # compile flags for C compiler -CFLAGS = @CFLAGS@ +CFLAGS.g = @CFLAGSG@ +CFLAGS.o = @CFLAGSO@ # if in debug mode, add TBB assertions ifeq ($(enable-threads),yes) @@ -368,7 +372,12 @@ ifeq ($(USE_CONTRIB_PETSC),yes) include $(DEAL_II_PETSC_DIR)/bmake/common/variables endif else + # PETSC's $(PETSC_ARCH)/conf/petscvariables include file happens + # to have a variable $(CXX) of itself. we need to save our own + # variable and restore it later. this isn't pretty :-( + SAVE_CXX := $(CXX) include $(DEAL_II_PETSC_DIR)/conf/variables + CXX := $(SAVE_CXX) endif CXXFLAGS.g += $(GCXX_PETSCFLAGS) CXXFLAGS.o += $(OCXX_PETSCFLAGS) diff --git a/deal.II/common/scripts/normalize.pl b/deal.II/common/scripts/normalize.pl index 03037323fe..ec90f06c20 100644 --- a/deal.II/common/scripts/normalize.pl +++ b/deal.II/common/scripts/normalize.pl @@ -1,7 +1,7 @@ ###################################################################### # $Id$ # -# Copyright (C) 2001, 2003, 2005, the deal.II authors +# Copyright (C) 2001, 2003, 2005, 2010, the deal.II authors # # Remove insignificant volatile data from output files of tests # @@ -48,3 +48,7 @@ s/-0\.00/0.00/g; #s/value.*//; #s/with residual.*//; + + +# remove deal.II debug output +s/^DEAL.*::_.*\n//g; diff --git a/deal.II/configure b/deal.II/configure index 929b20921f..d1c4a64e29 100755 --- a/deal.II/configure +++ b/deal.II/configure @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Revision: 22370 . +# From configure.in Revision: 22333 . # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.63 for deal.II 6.4.pre. # @@ -661,6 +661,9 @@ USE_CONTRIB_HSL HSL_INCLUDE_DIR NEEDS_F77LIBS DEAL_II_USE_BLAS +DEAL_II_P4EST_DIR +USE_CONTRIB_P4EST +DEAL_II_USE_P4EST DEAL_II_DEFINE_DEAL_II_USE_MUMPS DEAL_II_BLACS_ARCH DEAL_II_BLACS_DIR @@ -723,6 +726,8 @@ GREP CXXCPP enablethreads CFLAGSPIC +CFLAGSG +CFLAGSO CC_VERSION SHLIBFLAGS SHLIBLD @@ -799,6 +804,7 @@ SHELL' ac_subst_files='' ac_user_opts=' enable_option_checking +enable_mpi enable_threads enable_shared enable_parser @@ -815,6 +821,7 @@ with_arpack with_mumps with_scalapack with_blacs +with_p4est with_blas with_zlib with_netcdf @@ -1467,6 +1474,8 @@ Optional Features: --disable-option-checking ignore unrecognized --enable/--with options --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] + --enable-mpi Select MPI-enabled compilers and MPI support in + deal.II. --enable-threads Use multiple threads inside deal.II --enable-shared Set compiler flags to generate shared libraries --enable-parser While switched on by default, this option allows to @@ -1532,6 +1541,8 @@ Optional Packages: Specify the path to the BLACS installation; use this if you want to override the BLACS_DIR environment variable. + --with-p4est=/path/to/p4est makes deal.II use p4est to distribute meshes + on a cluster computer --with-blas=blaslib Use the blas library blaslib. Make sure the path to the libary is searched by ld, since it is included by the argument -lblaslib. If no argument @@ -2251,6 +2262,24 @@ $as_echo "" >&6; } { $as_echo "$as_me:$LINENO: result: ---------------- configuring C/C++ compilers ----------------" >&5 $as_echo "---------------- configuring C/C++ compilers ----------------" >&6; } + +# Check whether --enable-mpi was given. +if test "${enable_mpi+set}" = set; then + enableval=$enable_mpi; + { $as_echo "$as_me:$LINENO: checking whether to explicitly use MPI" >&5 +$as_echo_n "checking whether to explicitly use MPI... " >&6; } + if test "x$enableval" = "xyes" ; then + { $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + if test "x$CXX" = "x" ; then CXX=mpiCC ; fi + if test "x$CC" = "x" ; then CC=mpicc ; fi + else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } + fi +fi + + OLDCFLAGS="$CFLAGS" ac_ext=c ac_cpp='$CPP $CPPFLAGS' @@ -3950,6 +3979,10 @@ cat >>confdefs.h <<\_ACEOF #define DEAL_II_COMPILER_SUPPORTS_MPI 1 _ACEOF + + DEAL_II_COMPILER_SUPPORTS_MPI=1 + export DEAL_II_COMPILER_SUPPORTS_MPI + DEAL_II_USE_MPI=yes else @@ -5424,7 +5457,8 @@ $as_echo "Unrecognized compiler -- still trying" >&6; } if test "$GCC" = yes ; then - CFLAGS="$CFLAGS -O3 -funroll-loops -funroll-all-loops -fstrict-aliasing" + CFLAGSO="$CFLAGS -O3 -funroll-loops -funroll-all-loops -fstrict-aliasing" + CFLAGSG="$CFLAGS -g" case "$target" in *aix* ) CFLAGSPIC= @@ -5454,18 +5488,18 @@ $as_echo "Unrecognized compiler -- still trying" >&6; } case "$CC_VERSION" in ibm_xlc) - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGS -O2" CFLAGSPIC="-fPIC" SHLIBLD="$CXX" ;; MIPSpro*) - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGS -O2" CFLAGSPIC="-KPIC" ;; intel_icc*) - CFLAGS="$CFLAGS -O2 -unroll" + CFLAGSO="$CFLAGS -O2 -unroll" case "$CC_VERSION" in intel_icc5 | intel_icc6 | intel_icc7 | intel_icc8 | intel_icc9) CFLAGSPIC="-KPIC" @@ -5487,7 +5521,7 @@ $as_echo "Unrecognized compiler -- still trying" >&6; } ;; esac - CFLAGS="$CFLAGS -ansi_alias -vec_report0" + CFLAGSO="$CFLAGSO -ansi_alias -vec_report0" case "$target" in *86*) @@ -5578,7 +5612,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext *) { $as_echo "$as_me:$LINENO: result: Unknown C compiler - using generic options" >&5 $as_echo "Unknown C compiler - using generic options" >&6; } - CFLAGS="$CFLAGS -O2" + CFLAGSO="$CFLAGSO -O2" ;; esac fi @@ -5590,6 +5624,7 @@ $as_echo "Unknown C compiler - using generic options" >&6; } + # Check whether --enable-threads was given. if test "${enable_threads+set}" = set; then enableval=$enable_threads; @@ -12135,7 +12170,8 @@ $as_echo "$as_me: WARNING: Shared libraries not supported on $target. Using stat esac if test "x$enableshared" = "xyes" ; then - CFLAGS="$CFLAGS $CFLAGSPIC" + CFLAGSO="$CFLAGSO $CFLAGSPIC" + CFLAGSG="$CFLAGSG $CFLAGSPIC" CXXFLAGSG="$CXXFLAGSG $CXXFLAGSPIC" CXXFLAGSO="$CXXFLAGSO $CXXFLAGSPIC" F77FLAGSG="$F77FLAGSG $F77FLAGSPIC" @@ -14483,6 +14519,55 @@ _ACEOF + + { $as_echo "$as_me:$LINENO: checking whether p4est shall be used" >&5 +$as_echo_n "checking whether p4est shall be used... " >&6; } + + +# Check whether --with-p4est was given. +if test "${with_p4est+set}" = set; then + withval=$with_p4est; use_p4est=$withval +else + use_p4est=no +fi + + + if test "x$use_p4est" != "xno" ; then + { $as_echo "$as_me:$LINENO: result: yes" >&5 +$as_echo "yes" >&6; } + + if test ! -d "${use_p4est}/DEBUG" -o ! -d "${use_p4est}/FAST" ; then + echo "${use_p4est}/DEBUG" + { { $as_echo "$as_me:$LINENO: error: p4est directories $use_p4est/DEBUG or $use_p4est/FAST not found" >&5 +$as_echo "$as_me: error: p4est directories $use_p4est/DEBUG or $use_p4est/FAST not found" >&2;} + { (exit 1); exit 1; }; } + fi + + +cat >>confdefs.h <<\_ACEOF +#define DEAL_II_USE_P4EST 1 +_ACEOF + + USE_CONTRIB_P4EST=yes + export USE_CONTRIB_P4EST + + DEAL_II_P4EST_DIR=${use_p4est} + export DEAL_II_P4EST_DIR + + CXXFLAGSG="$CXXFLAGSG -I$use_p4est/DEBUG/include" + CXXFLAGSO="$CXXFLAGSO -I$use_p4est/FAST/include" + else + { $as_echo "$as_me:$LINENO: result: no" >&5 +$as_echo "no" >&6; } + fi + + + + + + + + if test "x$with_umfpack" != "x" -a "x$with_umfpack" != "xno" ; then if test "x$with_blas" = "x" -o "x$with_blas" = "xno"; then with_blas="yes" diff --git a/deal.II/configure.in b/deal.II/configure.in index 2aad166205..dae6764348 100644 --- a/deal.II/configure.in +++ b/deal.II/configure.in @@ -83,6 +83,22 @@ dnl ------------------------------------------------------------- AC_MSG_RESULT() AC_MSG_RESULT(---------------- configuring C/C++ compilers ----------------) + +dnl See if the user has specified --enable-mpi. If so, and if $CXX and $CC have +dnl not been set to a particular value, then override them with 'mpiCC' and 'mpicc'. +AC_ARG_ENABLE(mpi, + AS_HELP_STRING([--enable-mpi], + [Select MPI-enabled compilers and MPI support in deal.II.]), + [ + AC_MSG_CHECKING(whether to explicitly use MPI) + if test "x$enableval" = "xyes" ; then + AC_MSG_RESULT(yes) + if test "x$CXX" = "x" ; then CXX=mpiCC ; fi + if test "x$CC" = "x" ; then CC=mpicc ; fi + else + AC_MSG_RESULT(no) + fi]) + dnl Find a C compiler. This modifies the variable CC. dnl In order to get the absolute path of the compiler, use the dnl second line @@ -132,7 +148,8 @@ DEAL_II_DETERMINE_CC_BRAND DEAL_II_SET_CC_FLAGS AC_SUBST(CC_VERSION) -AC_SUBST(CFLAGS) +AC_SUBST(CFLAGSO) +AC_SUBST(CFLAGSG) AC_SUBST(CFLAGSPIC) dnl ------------------------------------------------------------- @@ -296,7 +313,8 @@ case "$target" in esac if test "x$enableshared" = "xyes" ; then - CFLAGS="$CFLAGS $CFLAGSPIC" + CFLAGSO="$CFLAGSO $CFLAGSPIC" + CFLAGSG="$CFLAGSG $CFLAGSPIC" CXXFLAGSG="$CXXFLAGSG $CXXFLAGSPIC" CXXFLAGSO="$CXXFLAGSO $CXXFLAGSPIC" F77FLAGSG="$F77FLAGSG $F77FLAGSPIC" @@ -447,6 +465,14 @@ AC_SUBST(DEAL_II_BLACS_DIR) dnl MUMPS dependency AC_SUBST(DEAL_II_BLACS_ARCH) AC_SUBST(DEAL_II_DEFINE_DEAL_II_USE_MUMPS) +DEAL_II_CONFIGURE_P4EST +AC_SUBST(DEAL_II_USE_P4EST) +AC_SUBST(USE_CONTRIB_P4EST) +AC_SUBST(DEAL_II_P4EST_DIR) + + + + dnl Make sure we configure for libraries used by other libraries. For dnl example, UMFPACK needs blas, and so does LAPACK. if test "x$with_umfpack" != "x" -a "x$with_umfpack" != "xno" ; then diff --git a/deal.II/contrib/Makefile.in b/deal.II/contrib/Makefile.in index 8772b62aab..613a5832a1 100644 --- a/deal.II/contrib/Makefile.in +++ b/deal.II/contrib/Makefile.in @@ -22,6 +22,7 @@ hsl-clean: endif + ifeq ($(USE_CONTRIB_UMFPACK),yes) umfpack: @cd umfpack ; $(MAKE) @@ -30,6 +31,7 @@ umfpack: endif + ifeq ($(enable-parser),yes) functionparser: @cd functionparser ; $(MAKE) @@ -54,3 +56,4 @@ clean: hsl-clean tbb-clean @-rm -f ../lib/contrib/*/*.o .PHONY: default hsl clean hsl-clean umfpack tbb tbb-clean functionparser + diff --git a/deal.II/contrib/configure.in b/deal.II/contrib/configure.in index 0549b475e2..2df5296e43 100644 --- a/deal.II/contrib/configure.in +++ b/deal.II/contrib/configure.in @@ -30,7 +30,6 @@ dnl ------------------------------------------------------------- AC_CONFIG_SUBDIRS(utilities) - dnl ------------------------------------------------------------- dnl Output results dnl ------------------------------------------------------------- diff --git a/deal.II/contrib/hsl/Makefile b/deal.II/contrib/hsl/Makefile index 7783f95244..fe35c28982 100644 --- a/deal.II/contrib/hsl/Makefile +++ b/deal.II/contrib/hsl/Makefile @@ -1,5 +1,5 @@ # $Id$ -# Copyright (C) 2001, 2002, 2003, 2004, 2006, 2008, 2009 by Wolfgang Bangerth +# Copyright (C) 2001, 2002, 2003, 2004, 2006, 2008, 2009, 2010 by Wolfgang Bangerth D = ../.. @@ -54,7 +54,9 @@ $(LIBDIR)/bin/detached_ma27$(EXEEXT): \ clean: - -rm -f *~ lib/lib* lib/o/*.$(OBJEXT) lib/Makefile.dep + -rm -f $(LIBDIR)/contrib/hsl/*.$(OBJEXT) $(LIBDIR)/libhsl$(lib-suffix) \ + $(LIBDIR)/bin/detached_ma27$(EXEEXT) \ + Makefile.dep diff --git a/deal.II/contrib/umfpack/AMD/Source/Makefile b/deal.II/contrib/umfpack/AMD/Source/Makefile index 63d45e000a..8aebbe4a64 100644 --- a/deal.II/contrib/umfpack/AMD/Source/Makefile +++ b/deal.II/contrib/umfpack/AMD/Source/Makefile @@ -26,15 +26,15 @@ AMDL = $(addprefix $D/lib/contrib/umfpack/, $(addsuffix .$(OBJEXT), $(subst amd_ $D/lib/contrib/umfpack/amd_i_%.$(OBJEXT): amd_%.c $(INC) @echo =====umfpack==========optimized====== $( +#include +#include + +#include +#include + +DEAL_II_NAMESPACE_OPEN + +namespace parallel +{ + namespace distributed + { + // forward declarations + template class Triangulation; + + +/** + * Collection of functions controlling refinement and coarsening of + * parallel::distributed::Triangulation objects. This namespace + * provides similar functionality to the dealii::GridRefinement + * namespace, except that it works for meshes that are parallel and + * distributed. + * + * @ingroup grid + * @author Wolfgang Bangerth, 2009 + */ + namespace GridRefinement + { + /** + * Like + * dealii::GridRefinement::refine_and_coarsen_fixed_number, + * but for parallel distributed + * triangulation. + * + * The vector of criteria needs + * to be a vector of refinement + * criteria for all cells + * active on the current + * triangulation, + * i.e. tria.dealii::Triangulation::n_active_cells(). However, + * the function will only look + * at the indicators that + * correspond to those cells + * that are actually locally + * owned, and ignore the + * indicators for all other + * cells. The function will + * then coordinate among all + * processors that store part + * of the triangulation so that + * at the end @p + * top_fraction_of_cells are + * refined, where the fraction + * is enforced as a fraction of + * Triangulation::n_global_active_cells, + * not + * Triangulation::n_locally_active_cells + * on each processor + * individually. In other + * words, it may be that on + * some processors, no cells + * are refined at all. + * + * The same is true for the fraction of + * cells that is coarsened. + */ + template + void + refine_and_coarsen_fixed_number ( + parallel::distributed::Triangulation &tria, + const Vector &criteria, + const double top_fraction_of_cells, + const double bottom_fraction_of_cells); + + /** + * Like + * dealii::GridRefinement::refine_and_coarsen_fixed_fraction, + * but for parallel distributed + * triangulation. + * + * The vector of criteria needs to be a + * vector of refinement criteria for + * all cells active on the current + * triangulation, + * i.e. tria.dealii::Triangulation::n_active_cells(). However, + * the function will only look at the + * indicators that correspond to those + * cells that are actually locally + * owned, and ignore the indicators for + * all other cells. The function will + * then coordinate among all processors + * that store part of the triangulation + * so that at the end the smallest + * fraction of + * Triangulation::n_global_active_cells + * (not + * Triangulation::n_locally_active_cells + * on each processor individually) is + * refined that together make up a + * total of @p top_fraction_of_error of + * the total error. In other words, it + * may be that on some processors, no + * cells are refined at all. + * + * The same is true for the fraction of + * cells that is coarsened. + */ + template + void + refine_and_coarsen_fixed_fraction ( + parallel::distributed::Triangulation &tria, + const Vector &criteria, + const double top_fraction_of_error, + const double bottom_fraction_of_error); + } + } +} + + +DEAL_II_NAMESPACE_CLOSE + +#endif //__deal2__distributed_grid_refinement_h diff --git a/deal.II/deal.II/include/distributed/solution_transfer.h b/deal.II/deal.II/include/distributed/solution_transfer.h new file mode 100644 index 0000000000..af3bee627c --- /dev/null +++ b/deal.II/deal.II/include/distributed/solution_transfer.h @@ -0,0 +1,187 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- +#ifndef __deal2__distributed_solution_transfer_h +#define __deal2__distributed_solution_transfer_h + +#include +#include +#include + +#include + + +DEAL_II_NAMESPACE_OPEN + +namespace parallel +{ + + namespace distributed + { +/** + * Transfers a discrete FE function (like a solution vector) by + * interpolation while refining and/or + * coarsening a distributed grid and + * handles the necessary communication. + * + *

Usage

+ * @verbatim + * SolutionTransfer > soltrans(dof_handler); + * // flag some cells for refinement + * // and coarsening, e.g. + * GridRefinement::refine_and_coarsen_fixed_fraction( + * *tria, error_indicators, 0.3, 0.05); + * // prepare the triangulation, + * tria->prepare_coarsening_and_refinement(); + * // prepare the SolutionTransfer object + * // for coarsening and refinement and give + * // the solution vector that we intend to + * // interpolate later, + * soltrans.prepare_for_coarsening_and_refinement(solution); + * // actually execute the refinement, + * tria->execute_coarsening_and_refinement (); + * // redistribute dofs, + * dof_handler->distribute_dofs (fe); + * // and interpolate the solution + * Vector interpolate_solution(dof_handler->n_dofs()); + * soltrans.interpolate(solution, interpolated_solution); + * @endverbatim + * @ingroup distributed + * @author Timo Heister, 2009 + */ + template > + class SolutionTransfer + { + public: + /** + * Constructor, takes the current DoFHandler + * as argument. + */ + SolutionTransfer(const DH &dof); + /** + * Destructor. + */ + ~SolutionTransfer(); + + /** + * Prepares the @p SolutionTransfer for + * coarsening and refinement. It + * stores the dof indices of each cell and + * stores the dof values of the vectors in + * @p all_in in each cell that'll be coarsened. + * @p all_in includes all vectors + * that are to be interpolated + * onto the new (refined and/or + * coarsenend) grid. + */ + void prepare_for_coarsening_and_refinement (const std::vector &all_in); + + /** + * Same as previous function + * but for only one discrete function + * to be interpolated. + */ + void prepare_for_coarsening_and_refinement (const VECTOR &in); + + /** + * + */ + void interpolate (std::vector &all_out); + + /** + * Same as the previous function. + * It interpolates only one function. + * It assumes the vectors having the + * right sizes (i.e. in.size()==n_dofs_old, + * out.size()==n_dofs_refined) + * + * Multiple calling of this function is + * NOT allowed. Interpolating + * several functions can be performed + * in one step by using + * interpolate (all_in, all_out) + */ + void interpolate (VECTOR &out); + + + /** + * return the size in bytes that need + * to be stored per cell. + */ + unsigned int get_data_size() const; + + + + private: + /** + * Pointer to the degree of + * freedom handler to work + * with. + */ + SmartPointer > dof_handler; + + /** + * A vector that stores + * pointers to all the + * vectors we are supposed to + * copy over from the old to + * the new mesh. + */ + std::vector input_vectors; + + /** + * The offset that the + * Triangulation has assigned + * to this object starting at + * which we are allowed to + * write. + */ + unsigned int offset; + + /** + * A callback function used + * to pack the data on the + * current mesh into objects + * that can later be + * retrieved after + * refinement, coarsening and + * repartitioning. + */ + void pack_callback(const typename Triangulation::cell_iterator &cell, + const typename Triangulation::CellStatus status, + void* data); + + /** + * A callback function used + * to unpack the data on the + * current mesh that has been + * packed up previously on + * the mesh before + * refinement, coarsening and + * repartitioning. + */ + void unpack_callback(const typename Triangulation::cell_iterator &cell, + const typename Triangulation::CellStatus status, + const void* data, + std::vector &all_out); + + }; + + + } +} + + + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/deal.II/deal.II/include/distributed/tria.h b/deal.II/deal.II/include/distributed/tria.h new file mode 100644 index 0000000000..c5f9deea39 --- /dev/null +++ b/deal.II/deal.II/include/distributed/tria.h @@ -0,0 +1,772 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- +#ifndef __deal2__distributed_tria_h +#define __deal2__distributed_tria_h + + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#ifdef DEAL_II_USE_P4EST +#include +#include +#include + +#include +#include +#include +#endif + + +DEAL_II_NAMESPACE_OPEN + +template class Triangulation; + +#ifdef DEAL_II_USE_P4EST + +namespace internal +{ + namespace DoFHandler + { + namespace Policy + { + template class ParallelDistributed; + } + } +} + + +namespace internal +{ + namespace p4est + { + /** + * A structure whose explicit + * specializations contain + * typedefs to the relevant + * p4est_* and p8est_* + * types. Using this + * structure, for example by + * saying + * types@::connectivity + * we can write code in a + * dimension independent way, + * either referring to + * p4est_connectivity_t or + * p8est_connectivity_t, + * depending on template + * argument. + */ + template struct types; + + template <> + struct types<2> + { + typedef p4est_connectivity_t connectivity; + typedef p4est_t forest; + typedef p4est_tree_t tree; + typedef p4est_quadrant_t quadrant; + typedef p4est_topidx_t topidx; + typedef p4est_locidx_t locidx; + typedef p4est_balance_type_t balance_type; + typedef p4est_ghost_t ghost; + }; + + template <> + struct types<3> + { + typedef p8est_connectivity_t connectivity; + typedef p8est_t forest; + typedef p8est_tree_t tree; + typedef p8est_quadrant_t quadrant; + typedef p4est_topidx_t topidx; + typedef p4est_locidx_t locidx; + typedef p8est_balance_type_t balance_type; + typedef p8est_ghost_t ghost; + }; + + + /** + * Initialize the + * GeometryInfo::max_children_per_cell + * children of the cell + * p4est_cell. + */ + template + void + init_quadrant_children + (const typename types::quadrant & p4est_cell, + typename types::quadrant (&p4est_children)[GeometryInfo::max_children_per_cell]); + + + /** + * Initialize quadrant to represent a coarse cell. + */ + template + void + init_coarse_quadrant(typename types::quadrant & quad); + + + + /** + * Returns whether q1 and q2 are equal + */ + template + bool + quadrant_is_equal (const typename types::quadrant & q1, + const typename types::quadrant & q2); + + //TODO: remove these functions from + //public interface somehow? [TH] + + /** + * returns whether q1 is an ancestor of q2 + */ + template + bool + quadrant_is_ancestor (const typename types::quadrant & q1, + const typename types::quadrant & q2); + } +} + + + +namespace parallel +{ + namespace distributed + { + + +/** + * This class acts like the dealii::Triangulation class, but it + * distributes the mesh across a number of different processors when + * using MPI. The class's interface does not add a lot to the + * dealii::Triangulation class but there are a number of difficult + * algorithms under the hood that ensure we always have a + * load-balanced, fully distributed mesh. Use of this class is + * explained in step-40, step-32, the @ref distributed documentation + * module, as well as the @ref distributed_paper . See there for more + * information. + * + * @author Wolfgang Bangerth, Timo Heister 2008, 2009, 2010 + * @ingroup distributed + */ + template + class Triangulation : public dealii::Triangulation + { + public: + /** + * Import the various + * iterator typedefs from the + * base class. + */ + typedef typename dealii::Triangulation::active_cell_iterator active_cell_iterator; + typedef typename dealii::Triangulation::cell_iterator cell_iterator; + typedef typename dealii::Triangulation::raw_cell_iterator raw_cell_iterator; + + /** + * Constructor. + * + * @param mpi_communicator denotes + * the MPI communicator to be used for + * the triangulation. + * + * @param smooth_grid Degree + * and kind of mesh smoothing + * to be applied to the + * mesh. See the + * dealii::Triangulation + * class for a description of + * the kinds of smoothing + * operations that can be + * applied. + * + * @note This class does not + * currently support the + * check_for_distorted_cells + * argument provided by the + * base class. + */ + Triangulation (MPI_Comm mpi_communicator, + const typename dealii::Triangulation::MeshSmoothing + smooth_grid = (dealii::Triangulation::none)); + + /** + * Destructor. + */ + virtual ~Triangulation (); + + /** + * Reset this triangulation into a + * virgin state by deleting all data. + * + * Note that this operation is only + * allowed if no subscriptions to this + * object exist any more, such as + * DoFHandler objects using it. + */ + virtual void clear (); + + /** + * Implementation of the same + * function as in the base + * class. + */ + virtual void copy_triangulation (const dealii::Triangulation &old_tria); + + /** + * Create a triangulation as + * documented in the base + * class. + * + * This function also sets up + * the various data + * structures necessary to + * distribute a mesh across a + * number of processors. This + * will be necessary once the + * mesh is being refined, + * though we will always keep + * the entire coarse mesh + * that is generated by this + * function on all + * processors. + */ + virtual void create_triangulation (const std::vector > &vertices, + const std::vector > &cells, + const SubCellData &subcelldata); + + /** + * Coarsen and refine the + * mesh according to + * refinement and coarsening + * flags set. + * + * Since the current + * processor only has control + * over those cells it owns + * (i.e. the ones for which + * cell-@>subdomain_id() + * == + * this-@>locally_owned_subdomain()), + * refinement and coarsening + * flags are only respected + * for those locally owned + * cells. Flags may be set on + * other cells as well (and + * may often, in fact, if you + * call + * Triangulation::prepare_coarsening_and_refinement) + * but will be largely + * ignored: the decision to + * refine the global mesh + * will only be affected by + * flags set on locally owned + * cells. + */ + virtual void execute_coarsening_and_refinement (); + + /** + * Return the subdomain id of + * those cells that are owned + * by the current + * processor. All cells in + * the triangulation that do + * not have this subdomain id + * are either owned by + * another processor or have + * children that only exist + * on other processors. + */ + types::subdomain_id_t locally_owned_subdomain () const; + + /** + * Return the number of + * active cells in the + * triangulation that are + * locally owned, i.e. that + * have a subdomain_id equal + * to + * locally_owned_subdomain(). Note + * that there may be more + * active cells in the + * triangulation stored on + * the present processor, + * such as for example ghost + * cells, or cells further + * away from the locally + * owned block of cells but + * that are needed to ensure + * that the triangulation + * that stores this + * processor's set of active + * cells still remains + * balanced with respect to + * the 2:1 size ratio of + * adjacent cells. + * + * As a consequence of the + * remark above, the result + * of this function is always + * smaller or equal to the + * result of + * ::Triangulation::n_active_cells() + * which includes the active + * ghost and artificial cells + * (see also @ref + * GlossArtificialCell and + * @ref GlossGhostCell). + */ + unsigned int n_locally_owned_active_cells () const; + + /** + * Return the sum over all + * processors of the number + * of active cells owned by + * each processor. This + * equals the overall number + * of active cells in the + * distributed triangulation. + */ + unsigned int n_global_active_cells () const; + + /** + * Return the number of + * active cells owned by each + * of the MPI processes that + * contribute to this + * triangulation. The element + * of this vector indexed by + * locally_owned_subdomain() + * equals the result of + * n_locally_owned_active_cells(). + */ + const std::vector & + n_locally_owned_active_cells_per_processor () const; + + /** + * Return the MPI + * communicator used by this + * triangulation. + */ + MPI_Comm get_communicator () const; + + /** + * Return the local memory + * consumption in bytes. + */ + virtual unsigned int memory_consumption () const; + + /** + * Return the local memory + * consumption contained in the p4est + * data structures alone. This is + * already contained in + * memory_consumption() but made + * available separately for debugging + * purposes. + */ + virtual unsigned int memory_consumption_p4est () const; + + /** + * A collective operation that produces + * a sequence of output files with the + * given file base name that contain + * the mesh in VTK format. + * + * More than anything else, this + * function is useful for debugging the + * interface between deal.II and p4est. + */ + void write_mesh_vtk (const char *file_basename) const; + + /** + * Produce a check sum of the + * triangulation. This is a + * collective operation and + * is mostly useful for + * debugging purposes. + */ + unsigned int get_checksum () const; + + /** + * Used to inform in the callbacks of + * register_data_attach() and + * notify_ready_to_unpack() how the + * cell with the given cell_iterator + * is going to change. Note that + * this may me different then the + * refine_flag() and coarsen_flag() + * in the cell_iterator because of + * refinement constraints that this + * machine does not see. + */ + enum CellStatus + { + CELL_PERSIST, CELL_REFINE, CELL_COARSEN, CELL_INVALID + }; + + /** + * Register a function with + * the current Triangulation + * object that will be used + * to attach data to active + * cells before + * execute_coarsening_and_refinement(). In + * execute_coarsening_and_refinement() + * the Triangulation will + * call the given function + * pointer and provide + * @p size bytes to store + * data. If necessary, this data will be + * transferred to the new + * owner of that cell during repartitioning + * the tree. See + * notify_ready_to_unpack() + * on how to retrieve the + * data. + * + * Callers need to store the + * return value. It + * specifies an offset of the + * position at which data can + * later be retrieved during + * a call to + * notify_ready_to_unpack(). + */ + unsigned int + register_data_attach (const size_t size, + const std_cxx1x::function & pack_callback); + + /** + * The given function is called for + * each new active cell and supplies + * a pointer to the data saved with + * register_data_attach(). + */ + void + notify_ready_to_unpack (const unsigned int offset, + const std_cxx1x::function & unpack_callback); + + private: + /** + * MPI communicator to be + * used for the + * triangulation. We create a + * unique communicator for + * this class, which is a + * duplicate of the one + * passed to the constructor. + */ + MPI_Comm mpi_communicator; + + /** + * The subdomain id to be + * used for the current + * processor. + */ + types::subdomain_id_t my_subdomain; + + /** + * A flag that indicates whether the + * triangulation has actual content. + */ + bool triangulation_has_content; + + /** + * A structure that contains + * some numbers about the + * distributed triangulation. + */ + struct NumberCache + { + std::vector n_locally_owned_active_cells; + unsigned int n_global_active_cells; + }; + + NumberCache number_cache; + + /** + * A data structure that holds the + * connectivity between trees. Since + * each tree is rooted in a coarse grid + * cell, this data structure holds the + * connectivity between the cells of + * the coarse grid. + */ + typename dealii::internal::p4est::types::connectivity *connectivity; + + /** + * A data structure that holds the + * local part of the global + * triangulation. + */ + typename dealii::internal::p4est::types::forest *parallel_forest; + + /** + * A flag that indicates + * whether refinement of a + * triangulation is currently + * in progress. We need this + * flag because the + * refinement_notification() + * function is called from + * Triangulation::execute_coarsening_and_refinement(), + * but + * refinement_notification() + * itself also calls + * Triangulation::execute_coarsening_and_refinement() + * and therefore gets called + * recursively again. While the + * first time we want to take + * over work to copy things + * from a refined p4est, the + * other times we don't want to + * get in the way as these + * latter calls to + * Triangulation::execute_coarsening_and_refinement() + * are simply there in order to + * re-create a triangulation + * that matches the p4est. + */ + bool refinement_in_progress; + + + /** + * + */ + unsigned int attached_data_size; + unsigned int n_attached_datas; + typedef std_cxx1x::function< + void(typename Triangulation::cell_iterator, CellStatus, void*) + > pack_callback_t; + + typedef std::pair callback_pair_t; + + typedef std::list callback_list_t; + callback_list_t attached_data_pack_callbacks; + + + + + + /** + * Two arrays that store which p4est + * tree corresponds to which coarse + * grid cell and vice versa. We need + * these arrays because p4est goes with + * the original order of coarse cells + * when it sets up its forest, and then + * applies the Morton ordering within + * each tree. But if coarse grid cells + * are badly ordered this may mean that + * individual parts of the forest + * stored on a local machine may be + * split across coarse grid cells that + * are not geometrically + * close. Consequently, we apply a + * Cuthill-McKee preordering to ensure + * that the part of the forest stored + * by p4est is located on geometrically + * close coarse grid cells. + */ + std::vector coarse_cell_to_p4est_tree_permutation; + std::vector p4est_tree_to_coarse_cell_permutation; + + /** + * Return a pointer to the p4est + * tree that belongs to the given + * dealii_coarse_cell_index() + */ + typename dealii::internal::p4est::types::tree * + init_tree(const int dealii_coarse_cell_index) const; + + /** + * The function that computes the + * permutation between the two data + * storage schemes. + */ + void setup_coarse_cell_to_p4est_tree_permutation (); + + /** + * Take the contents of a newly created + * triangulation we are attached to and + * copy it to p4est data structures. + * + * This function exists in 2d + * and 3d variants. + */ + void copy_new_triangulation_to_p4est (dealii::internal::int2type<2>); + void copy_new_triangulation_to_p4est (dealii::internal::int2type<3>); + + /** + * Copy the local part of the refined + * forest from p4est into the attached + * triangulation. + */ + void copy_local_forest_to_triangulation (); + + + /** + * Update the number_cache + * variable after mesh + * creation or refinement. + */ + void update_number_cache (); + + /** + * Internal function notifying all + * registered classes to attach their + * data before repartitioning + * occurs. Called from + * execute_coarsening_and_refinement(). + */ + void attach_mesh_data(); + + + template friend class dealii::internal::DoFHandler::Policy::ParallelDistributed; + }; + + + /** + * Specialization of the general template + * for the 1d case. There is currently no + * support for distributing 1d + * triangulations. Consequently, all this + * class does is throw an exception. + */ + template + class Triangulation<1,spacedim> : public dealii::Triangulation<1,spacedim> + { + public: + /** + * Constructor. The argument denotes + * the MPI communicator to be used for + * the triangulation. + */ + Triangulation (MPI_Comm mpi_communicator); + + /** + * Destructor. + */ + virtual ~Triangulation (); + + /** + * Return the MPI + * communicator used by this + * triangulation. + */ + MPI_Comm get_communicator () const; + + /** + * Return the subdomain id of + * those cells that are owned + * by the current + * processor. All cells in + * the triangulation that do + * not have this subdomain id + * are either owned by + * another processor or have + * children that only exist + * on other processors. + */ + types::subdomain_id_t locally_owned_subdomain () const; + + /** + * Dummy arrays. This class + * isn't usable but the + * compiler wants to see + * these variables at a + * couple places anyway. + */ + std::vector coarse_cell_to_p4est_tree_permutation; + std::vector p4est_tree_to_coarse_cell_permutation; + }; + } +} + + +#else // DEAL_II_USE_P4EST + +namespace parallel +{ + namespace distributed + { + /** + * Dummy class the compiler chooses for + * parallel distributed triangulations if + * we didn't actually configure deal.II + * with the p4est library. The existence + * of this class allows us to refer to + * parallel::distributed::Triangulation + * objects throughout the library even if + * it is disabled. + * + * Since the constructor of this class is + * private, no such objects can actually + * be created if we don't have p4est + * available. + */ + template + class Triangulation : public dealii::Triangulation + { + private: + /** + * Constructor. + */ + Triangulation (); + + public: + + /** + * Destructor. + */ + virtual ~Triangulation (); + + /** + * Return the subdomain id of + * those cells that are owned + * by the current + * processor. All cells in + * the triangulation that do + * not have this subdomain id + * are either owned by + * another processor or have + * children that only exist + * on other processors. + */ + types::subdomain_id_t locally_owned_subdomain () const; + }; + } +} + + +#endif + + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/deal.II/deal.II/include/dofs/dof_accessor.h b/deal.II/deal.II/include/dofs/dof_accessor.h index f401d9bfba..1ab4b54a07 100644 --- a/deal.II/deal.II/include/dofs/dof_accessor.h +++ b/deal.II/deal.II/include/dofs/dof_accessor.h @@ -40,6 +40,10 @@ namespace internal namespace DoFHandler { struct Implementation; + namespace Policy + { + struct Implementation; + } } namespace hp @@ -51,8 +55,6 @@ namespace internal } } - - // note: the file dof_accessor.templates.h is included at the end of // this file. this includes a lot of templates and thus makes // compilation slower, but at the same time allows for more aggressive @@ -288,7 +290,7 @@ class DoFAccessor : public internal::DoFAccessor::Inheritance &da); - + /** * Return an iterator pointing to * the the parent. @@ -732,8 +734,10 @@ class DoFAccessor : public internal::DoFAccessor::Inheritance friend class DoFHandler; template friend class hp::DoFHandler; + friend class internal::DoFHandler::Policy::Implementation; friend class internal::DoFHandler::Implementation; friend class internal::hp::DoFHandler::Implementation; + friend class internal::DoFCellAccessor::Implementation; }; @@ -851,7 +855,7 @@ class DoFCellAccessor : public DoFAccessor */ typename internal::DoFHandler::Iterators::cell_iterator parent () const; - + /** * @name Accessing sub-objects and neighbors */ @@ -1382,7 +1386,15 @@ class DoFCellAccessor : public DoFAccessor * @} */ - private: + /** + * Set the DoF indices of this + * cell to the given values. This + * function bypasses the DoF + * cache, if one exists for the + * given DoF handler class. + */ + void set_dof_indices (const std::vector &dof_indices); + /** * Update the cache in which we * store the dof indices of this @@ -1397,7 +1409,6 @@ class DoFCellAccessor : public DoFAccessor * function */ template friend class DoFHandler; - friend class internal::DoFCellAccessor::Implementation; }; diff --git a/deal.II/deal.II/include/dofs/dof_accessor.templates.h b/deal.II/deal.II/include/dofs/dof_accessor.templates.h index f567fe5b2a..0768ac47f6 100644 --- a/deal.II/deal.II/include/dofs/dof_accessor.templates.h +++ b/deal.II/deal.II/include/dofs/dof_accessor.templates.h @@ -2049,6 +2049,163 @@ namespace internal Assert (false, ExcNotImplemented()); } + /** + * Implement setting dof + * indices on a + * cell. Currently not + * implemented for + * hp::DoFHandler objects. + */ + template + static + void + set_dof_indices (DoFCellAccessor > &accessor, + const std::vector &local_dof_indices) + { + Assert (accessor.has_children() == false, + ExcInternalError()); + + const unsigned int dofs_per_vertex = accessor.get_fe().dofs_per_vertex, + dofs_per_line = accessor.get_fe().dofs_per_line, + dofs_per_cell = accessor.get_fe().dofs_per_cell; + + Assert (local_dof_indices.size() == dofs_per_cell, + ExcInternalError()); + + unsigned int index = 0; + + for (unsigned int vertex=0; vertex<2; ++vertex) + for (unsigned int d=0; d + static + void + set_dof_indices (DoFCellAccessor > &accessor, + const std::vector &local_dof_indices) + { + Assert (accessor.has_children() == false, + ExcInternalError()); + + const unsigned int dofs_per_vertex = accessor.get_fe().dofs_per_vertex, + dofs_per_line = accessor.get_fe().dofs_per_line, + dofs_per_quad = accessor.get_fe().dofs_per_quad, + dofs_per_cell = accessor.get_fe().dofs_per_cell; + + Assert (local_dof_indices.size() == dofs_per_cell, + ExcInternalError()); + + unsigned int index = 0; + + for (unsigned int vertex=0; vertex<4; ++vertex) + for (unsigned int d=0; dset_dof_index(d, local_dof_indices[index]); + + for (unsigned int d=0; d + static + void + set_dof_indices (DoFCellAccessor > &accessor, + const std::vector &local_dof_indices) + { + Assert (accessor.has_children() == false, + ExcInternalError()); + + const unsigned int dofs_per_vertex = accessor.get_fe().dofs_per_vertex, + dofs_per_line = accessor.get_fe().dofs_per_line, + dofs_per_quad = accessor.get_fe().dofs_per_quad, + dofs_per_hex = accessor.get_fe().dofs_per_hex, + dofs_per_cell = accessor.get_fe().dofs_per_cell; + + Assert (local_dof_indices.size() == dofs_per_cell, + ExcInternalError()); + + unsigned int index = 0; + + for (unsigned int vertex=0; vertex<8; ++vertex) + for (unsigned int d=0; dset_dof_index(accessor.dof_handler->get_fe(). + adjust_line_dof_index_for_line_orientation(d, + accessor.line_orientation(line)), + local_dof_indices[index]); + // now copy dof numbers into the face. for + // faces with the wrong orientation, we + // have already made sure that we're ok by + // picking the correct lines and vertices + // (this happens automatically in the + // line() and vertex() functions). however, + // if the face is in wrong orientation, we + // look at it in flipped orientation and we + // will have to adjust the shape function + // indices that we see to correspond to the + // correct (cell-local) ordering. The same + // applies, if the face_rotation or + // face_orientation is non-standard + for (unsigned int quad=0; quad<6; ++quad) + for (unsigned int d=0; dset_dof_index(accessor.dof_handler->get_fe(). + adjust_quad_dof_index_for_face_orientation(d, + accessor.face_orientation(quad), + accessor.face_flip(quad), + accessor.face_rotation(quad)), + local_dof_indices[index]); + for (unsigned int d=0; d + static + void + set_dof_indices (const DoFCellAccessor > &, + const std::vector &) + { + Assert (false, ExcNotImplemented()); + } + /** * A function that collects the @@ -2832,6 +2989,8 @@ void DoFCellAccessor:: get_dof_indices (std::vector &dof_indices) const { + Assert (this->is_artificial() == false, + ExcMessage ("Can't ask for DoF indices on artificial cells.")); internal::DoFCellAccessor::Implementation::get_dof_indices (*this, dof_indices); } @@ -2843,6 +3002,8 @@ void DoFCellAccessor::get_dof_values (const InputVector &values, Vector &local_values) const { + Assert (this->is_artificial() == false, + ExcMessage ("Can't ask for DoF indices on artificial cells.")); internal::DoFCellAccessor::Implementation ::get_dof_values (*this, values, local_values.begin(), local_values.end()); } @@ -2856,6 +3017,8 @@ DoFCellAccessor::get_dof_values (const InputVector &values, ForwardIterator local_values_begin, ForwardIterator local_values_end) const { + Assert (this->is_artificial() == false, + ExcMessage ("Can't ask for DoF indices on artificial cells.")); internal::DoFCellAccessor::Implementation ::get_dof_values (*this, values, local_values_begin, local_values_end); } @@ -2870,6 +3033,8 @@ DoFCellAccessor::get_dof_values (const ConstraintMatrix &constraints, ForwardIterator local_values_begin, ForwardIterator local_values_end) const { + Assert (this->is_artificial() == false, + ExcMessage ("Can't ask for DoF indices on artificial cells.")); internal::DoFCellAccessor::Implementation ::get_dof_values (*this, constraints, values, local_values_begin, local_values_end); @@ -2883,6 +3048,8 @@ void DoFCellAccessor::set_dof_values (const Vector &local_values, OutputVector &values) const { + Assert (this->is_artificial() == false, + ExcMessage ("Can't ask for DoF indices on artificial cells.")); internal::DoFCellAccessor::Implementation ::set_dof_values (*this, local_values, values); } diff --git a/deal.II/deal.II/include/dofs/dof_handler.h b/deal.II/deal.II/include/dofs/dof_handler.h index 81f4992d79..52d9dd707a 100644 --- a/deal.II/deal.II/include/dofs/dof_handler.h +++ b/deal.II/deal.II/include/dofs/dof_handler.h @@ -18,9 +18,12 @@ #include #include #include +#include #include #include +#include #include +#include #include #include @@ -105,6 +108,19 @@ namespace internal * algorithms. * * + *

Interaction with distributed meshes

+ * + * Upon construction, this class takes a reference to a triangulation + * object. In most cases, this will be a reference to an object of + * type Triangulation, i.e. the class that represents triangulations + * that entirely reside on a single processor. However, it can also be + * of type parallel::distributed::Triangulation (see, for example, + * step-32, step-40 and in particular the @ref distributed module) in + * which case the DoFHandler object will proceed to only manage + * degrees of freedom on locally owned and ghost cells. This process + * is entirely transparent to the used. + * + * *

User defined renumbering schemes

* * The DoFRenumbering class offers a number of renumbering schemes like the @@ -131,7 +147,7 @@ class DoFHandler : public Subscriptor public: typedef typename IteratorSelector::CellAccessor cell_accessor; typedef typename IteratorSelector::FaceAccessor face_accessor; - + typedef typename IteratorSelector::raw_line_iterator raw_line_iterator; typedef typename IteratorSelector::line_iterator line_iterator; typedef typename IteratorSelector::active_line_iterator active_line_iterator; @@ -208,7 +224,7 @@ class DoFHandler : public Subscriptor * value. */ static const unsigned int default_fe_index = 0; - + /** * Standard constructor, not * initializing any data. After @@ -224,12 +240,12 @@ class DoFHandler : public Subscriptor * triangulation to work on. */ DoFHandler ( const Triangulation &tria); - + /** * Destructor. */ virtual ~DoFHandler (); - + /** * Assign a Triangulation and a * FiniteElement to the @@ -286,7 +302,7 @@ class DoFHandler : public Subscriptor * in the same object. */ void initialize_local_block_info(); - + /** * Clear all data of this object and * especially delete the lock this object @@ -300,31 +316,68 @@ class DoFHandler : public Subscriptor * a list of new dof numbers for all the * dofs. * - * @p new_numbers is an array of integers - * with size equal to the number of dofs - * on the present grid. It stores the new - * indices after renumbering in the - * order of the old indices. - * * This function is called by * the functions in * DoFRenumbering function * after computing the ordering * of the degrees of freedom. - * However, you can call this - * function yourself, which is - * necessary if a user wants to - * implement an ordering scheme - * herself, for example - * downwind numbering. + * This function is called, for + * example, by the functions in + * the DoFRenumbering + * namespace, but it can of + * course also be called from + * user code. * - * The @p new_number array must - * have a size equal to the + * @arg new_number This array + * must have a size equal to + * the number of degrees of + * freedom owned by the current + * processor, i.e. the size + * must be equal to what + * n_locally_owned_dofs() + * returns. If only one + * processor participates in + * storing the current mesh, + * then this equals the total * number of degrees of - * freedom. Each entry must - * state the new global DoF - * number of the degree of - * freedom referenced. + * freedom, i.e. the result of + * n_dofs(). The contents of + * this array are the new + * global indices for each + * freedom listed in the + * IndexSet returned by + * locally_owned_dofs(). In the + * case of a sequential mesh + * this means that the array is + * a list of new indices for + * each of the degrees of + * freedom on the current + * mesh. In the case that we + * have a + * parallel::distributed::Triangulation + * underlying this DoFHandler + * object, the array is a list + * of new indices for all the + * locally owned degrees of + * freedom, enumerated in the + * same order as the currently + * locally owned DoFs. In other + * words, assume that degree of + * freedom i is + * currently locally owned, + * then + * new_numbers[locally_owned_dofs().index_within_set(i)] + * returns the new global DoF + * index of + * i. Since the + * IndexSet of + * locally_owned_dofs() is + * complete in the sequential + * case, the latter convention + * for the content of the array + * reduces to the former in the + * case that only one processor + * participates in the mesh. */ void renumber_dofs (const std::vector &new_numbers); @@ -927,10 +980,35 @@ class DoFHandler : public Subscriptor /** - * Return number of degrees of freedom. - * Included in this number are those + * Return the global number of + * degrees of freedom. If the + * current object handles all + * degrees of freedom itself + * (even if you may intend to + * solve your linear system in + * parallel, such as in step-17 + * or step-18), then this number + * equals the number of locally + * owned degrees of freedom since + * this object doesn't know + * anything about what you want + * to do with it and believes + * that it owns every degree of + * freedom it knows about. + * + * On the other hand, if this + * object operates on a + * parallel::distributed::Triangulation + * object, then this function + * returns the global number of + * degrees of freedom, + * accumulated over all + * processors. + * + * In either case, included in + * the returned number are those * DoFs which are constrained by - * hanging nodes. + * hanging nodes, see @ref constraints. */ unsigned int n_dofs () const; @@ -994,7 +1072,99 @@ class DoFHandler : public Subscriptor * initialize_local_block_indices(). */ const BlockInfo& block_info() const; - + + + /** + * Return the number of + * degrees of freedom that + * belong to this + * process. + * + * If this is a sequential job, + * then the result equals that + * produced by n_dofs(). On the + * other hand, if we are + * operating on a + * parallel::distributed::Triangulation, + * then it includes only the + * degrees of freedom that the + * current processor owns. Note + * that in this case this does + * not include all degrees of + * freedom that have been + * distributed on the current + * processor's image of the mesh: + * in particular, some of the + * degrees of freedom on the + * interface between the cells + * owned by this processor and + * cells owned by other + * processors may be theirs, and + * degrees of freedom on ghost + * cells are also not necessarily + * included. + */ + unsigned int n_locally_owned_dofs() const; + + /** + * Return an IndexSet describing + * the set of locally owned DoFs + * as a subset of + * 0..n_dofs(). The number of + * elements of this set equals + * n_locally_owned_dofs(). + */ + const IndexSet & locally_owned_dofs() const; + + + /** + * Returns a vector that + * stores the locally owned + * DoFs of each processor. If + * you are only interested in + * the number of elements + * each processor owns then + * n_dofs_per_processor() is + * a better choice. + * + * If this is a sequential job, + * then the vector has a single + * element that equals the + * IndexSet representing the + * entire range [0,n_dofs()]. + */ + const std::vector & + locally_owned_dofs_per_processor () const; + + /** + * Return a vector that + * stores the number of + * degrees of freedom each + * processor that + * participates in this + * triangulation owns + * locally. The sum of all + * these numbers equals the + * number of degrees of + * freedom that exist + * globally, i.e. what + * n_dofs() returns. + * + * Each element of the vector + * returned by this function + * equals the number of + * elements of the + * corresponding sets + * returned by + * global_dof_indices(). + * + * If this is a sequential job, + * then the vector has a single + * element equal to n_dofs(). + */ + const std::vector & + n_locally_owned_dofs_per_processor () const; + /** * Return a constant reference to * the selected finite element @@ -1023,11 +1193,6 @@ class DoFHandler : public Subscriptor */ virtual unsigned int memory_consumption () const; - /** - * Exception - */ - DeclException0 (ExcInvalidTriangulation); - /** * @todo Replace by ExcInternalError. */ @@ -1072,22 +1237,30 @@ class DoFHandler : public Subscriptor int, << "You tried to do something on level " << arg1 << ", but this level is empty."); - - + + protected: /** * The object containing * information on the block structure. */ BlockInfo block_info_object; - - private: + + /** + * Array to store the indices for + * degrees of freedom located at + * vertices. + */ + std::vector vertex_dofs; + + /** * Address of the triangulation to * work on. */ - SmartPointer,DoFHandler > tria; + SmartPointer,DoFHandler > + tria; /** * Store a pointer to the finite element @@ -1101,7 +1274,30 @@ class DoFHandler : public Subscriptor * function (this clears all data of * this object as well, though). */ - SmartPointer,DoFHandler > selected_fe; + SmartPointer,DoFHandler > + selected_fe; + + /** + * An object that describes how degrees + * of freedom should be distributed and + * renumbered. + */ + std_cxx1x::shared_ptr > policy; + + /** + * A structure that contains all + * sorts of numbers that + * characterize the degrees of + * freedom this object works on. + * + * For most members of this + * structure, there is an + * accessor function in this + * class that returns its value. + */ + internal::DoFHandler::NumberCache number_cache; + + private: /** * Copy constructor. I can see no reason @@ -1149,19 +1345,6 @@ class DoFHandler : public Subscriptor */ internal::DoFHandler::DoFFaces *faces; - /** - * Store the number of dofs - * created last time. - */ - unsigned int used_dofs; - - /** - * Array to store the indices for - * degrees of freedom located at - * vertices. - */ - std::vector vertex_dofs; - /** * Make accessor objects friends. */ @@ -1171,6 +1354,7 @@ class DoFHandler : public Subscriptor friend class internal::DoFCellAccessor::Implementation; friend class internal::DoFHandler::Implementation; + friend class internal::DoFHandler::Policy::Implementation; }; @@ -1193,7 +1377,39 @@ inline unsigned int DoFHandler::n_dofs () const { - return used_dofs; + return number_cache.n_global_dofs; +} + + +template +unsigned int +DoFHandler::n_locally_owned_dofs() const +{ + return number_cache.n_locally_owned_dofs; +} + + +template +const IndexSet & +DoFHandler::locally_owned_dofs() const +{ + return number_cache.locally_owned_dofs; +} + + +template +const std::vector & +DoFHandler::n_locally_owned_dofs_per_processor() const +{ + return number_cache.n_locally_owned_dofs_per_processor; +} + + +template +const std::vector & +DoFHandler::locally_owned_dofs_per_processor () const +{ + return number_cache.locally_owned_dofs_per_processor; } diff --git a/deal.II/deal.II/include/dofs/dof_handler_policy.h b/deal.II/deal.II/include/dofs/dof_handler_policy.h new file mode 100644 index 0000000000..989ef73a07 --- /dev/null +++ b/deal.II/deal.II/include/dofs/dof_handler_policy.h @@ -0,0 +1,152 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- +#ifndef __deal2__dof_handler_policy_h +#define __deal2__dof_handler_policy_h + + + +#include +#include +#include + +#include +#include +#include + +DEAL_II_NAMESPACE_OPEN + +template class FiniteElement; +template class DoFHandler; + + +namespace internal +{ + namespace DoFHandler + { + class NumberCache; + + /** + * A namespace in which we define + * classes that describe how to + * distribute and renumber + * degrees of freedom. + */ + namespace Policy + { + struct Implementation; + + /** + * A class that implements policies for + * how the DoFHandler::distribute_dofs + * and DoFHandler::renumber_dofs + * functions should work. + */ + template + class PolicyBase + { + public: + /** + * Destructor. + */ + virtual ~PolicyBase (); + + /** + * Distribute degrees of freedom on + * the object given as last argument. + */ + virtual + NumberCache + distribute_dofs (const unsigned int offset, + dealii::DoFHandler &dof_handler) const = 0; + + /** + * Renumber degrees of freedom as + * specified by the first argument. + */ + virtual + NumberCache + renumber_dofs (const std::vector &new_numbers, + dealii::DoFHandler &dof_handler) const = 0; + }; + + + /** + * This class implements the + * default policy for sequential + * operations, i.e. for the case where + * all cells get degrees of freedom. + */ + template + class Sequential : public PolicyBase + { + public: + /** + * Distribute degrees of freedom on + * the object given as last argument. + */ + virtual + NumberCache + distribute_dofs (const unsigned int offset, + dealii::DoFHandler &dof_handler) const; + + /** + * Renumber degrees of freedom as + * specified by the first argument. + */ + virtual + NumberCache + renumber_dofs (const std::vector &new_numbers, + dealii::DoFHandler &dof_handler) const; + }; + + + /** + * This class implements the + * policy for operations when + * we use a + * parallel::distributed::Triangulation + * object. + */ + template + class ParallelDistributed : public PolicyBase + { + public: + /** + * Distribute degrees of freedom on + * the object given as last argument. + */ + virtual + NumberCache + distribute_dofs (const unsigned int offset, + dealii::DoFHandler &dof_handler) const; + + /** + * Renumber degrees of freedom as + * specified by the first argument. + */ + virtual + NumberCache + renumber_dofs (const std::vector &new_numbers, + dealii::DoFHandler &dof_handler) const; + }; + } + } +} + + + +DEAL_II_NAMESPACE_CLOSE + +/*---------------------------- dof_handler_policy.h ---------------------------*/ +#endif +/*---------------------------- dof_handler_policy.h ---------------------------*/ diff --git a/deal.II/deal.II/include/dofs/dof_renumbering.h b/deal.II/deal.II/include/dofs/dof_renumbering.h index d991afc881..152c2c8e1b 100644 --- a/deal.II/deal.II/include/dofs/dof_renumbering.h +++ b/deal.II/deal.II/include/dofs/dof_renumbering.h @@ -743,7 +743,6 @@ namespace DoFRenumbering component_wise (hp::DoFHandler &dof_handler, const std::vector &target_component = std::vector ()); - /** * Sort the degrees of freedom by * component. It does the same @@ -786,9 +785,9 @@ namespace DoFRenumbering * the renumbering vector. */ template - static unsigned int + unsigned int compute_component_wise (std::vector& new_dof_indices, - ITERATOR& start, + const ITERATOR& start, const ENDITERATOR& end, const std::vector &target_component); diff --git a/deal.II/deal.II/include/dofs/dof_tools.h b/deal.II/deal.II/include/dofs/dof_tools.h index 60fd39120d..bfc15afb6c 100644 --- a/deal.II/deal.II/include/dofs/dof_tools.h +++ b/deal.II/deal.II/include/dofs/dof_tools.h @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -457,7 +459,7 @@ class DoFTools SparsityPattern &sparsity_pattern, const ConstraintMatrix &constraints = ConstraintMatrix(), const bool keep_constrained_dofs = true, - const unsigned int subdomain_id = numbers::invalid_unsigned_int); + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id); /** * Locate non-zero entries for @@ -618,7 +620,7 @@ class DoFTools SparsityPattern &sparsity_pattern, const ConstraintMatrix &constraints = ConstraintMatrix(), const bool keep_constrained_dofs = true, - const unsigned int subdomain_id = numbers::invalid_unsigned_int); + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id); /** * @deprecated This is the old @@ -763,10 +765,10 @@ class DoFTools /** * This function does the same as * the other with the same name, - * but it gets a ConstraintMatrix - * additionally. - * This is for the case where you - * have fluxes but constraints as + * but it gets a ConstraintMatrix + * additionally. + * This is for the case where you + * have fluxes but constraints as * well. * Not implemented for * hp::DoFHandler. @@ -1169,9 +1171,81 @@ class DoFTools template static void extract_subdomain_dofs (const DH &dof_handler, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, std::vector &selected_dofs); + + /** + * Extract the set of global DoF + * indices that are owned by the + * current processor. For regular + * DoFHandler objects, this set + * is the complete set with all + * DoF indices. In either case, + * it equals what + * DoFHandler::locally_owned_dofs() + * returns. + */ + template + static void + extract_locally_owned_dofs (const DH & dof_handler, + IndexSet & dof_set); + + + /** + * Extract the set of global DoF + * indices that are active on the + * current DoFHandler. For + * regular DoFHandlers, these are + * all DoF indices, but for + * DoFHandler objects built on + * parallel::distributed::Triangulation + * this set is a superset of + * DoFHandler::locally_owned_dofs() + * and contains all DoF indices + * that live on all locally owned + * cells (including on the + * interface to ghost + * cells). However, it does not + * contain the DoF indices that + * are exclusively defined on + * ghost or artificial cells (see + * @ref GlossArtificialCell "the + * glossary"). + * + * The degrees of freedom identified by + * this function equal those obtained + * from the + * dof_indices_with_subdomain_association() + * function when called with the locally + * owned subdomain id. + */ + template + static void + extract_locally_active_dofs (const DH & dof_handler, + IndexSet & dof_set); + + /** + * Extract the set of global DoF + * indices that are active on the + * current DoFHandler. For + * regular DoFHandlers, these are + * all DoF indices, but for + * DoFHandler objects built on + * parallel::distributed::Triangulation + * this set is the union of + * DoFHandler::locally_owned_dofs() + * and the DoF indices on all + * ghost cells. In essence, it is + * the DoF indices on all cells + * that are not artificial (see + * @ref GlossArtificialCell "the glossary"). + */ + template + static void + extract_locally_relevant_dofs (const DH & dof_handler, + IndexSet & dof_set); + /** * Extract a vector that represents the * constant modes of the DoFHandler for @@ -1196,13 +1270,13 @@ class DoFTools * consist of as many vectors as there * are true arguments in * component_select, each of - * which will be one in one component and + * which will be one in one vector component and * zero in all others. We store this * object in a vector of vectors, where * the outer vector is of the size of the * number of selected components, and * each inner vector has as many - * components as there are degrees of + * components as there are (locally owned) degrees of * freedom in the selected * components. Note that any matrix * associated with this null space must @@ -1279,11 +1353,27 @@ class DoFTools * function, or use the * GridTools::get_subdomain_association * function. + * + * Note that this function is of + * questionable use for DoFHandler objects built on + * parallel::distributed::Triangulation + * since in that case ownership of + * individual degrees of freedom by MPI + * processes is controlled by the DoF + * handler object, not based on some + * geometric algorithm in conjunction + * with subdomain id. In particular, the + * degrees of freedom identified by the + * functions in this namespace as + * associated with a subdomain are not + * the same the + * DoFHandler class + * identifies as those it owns. */ template static void get_subdomain_association (const DH &dof_handler, - std::vector &subdomain); + std::vector &subdomain); /** * Count how many degrees of freedom are @@ -1307,25 +1397,27 @@ class DoFTools * @em cells with this subdomain, use the * GridTools::count_cells_with_subdomain_association * function. + * + * Note that this function is of + * questionable use for DoFHandler objects built on + * parallel::distributed::Triangulation + * since in that case ownership of + * individual degrees of freedom by MPI + * processes is controlled by the DoF + * handler object, not based on some + * geometric algorithm in conjunction + * with subdomain id. In particular, the + * degrees of freedom identified by the + * functions in this namespace as + * associated with a subdomain are not + * the same the + * DoFHandler class + * identifies as those it owns. */ template static unsigned int count_dofs_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain); - - /** - * Similar to the previous - * function, but do not just - * return the number of degrees - * of freedom that are owned by a - * given subdomain, but return a - * set of their indices. - */ - template - static - IndexSet - dof_indices_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain); + const types::subdomain_id_t subdomain); /** * Count how many degrees of freedom are @@ -1342,13 +1434,71 @@ class DoFTools * therefore store how many degrees of * freedom of each vector component are * associated with the given subdomain. + * + * Note that this function is of + * questionable use for DoFHandler objects built on + * parallel::distributed::Triangulation + * since in that case ownership of + * individual degrees of freedom by MPI + * processes is controlled by the DoF + * handler object, not based on some + * geometric algorithm in conjunction + * with subdomain id. In particular, the + * degrees of freedom identified by the + * functions in this namespace as + * associated with a subdomain are not + * the same the + * DoFHandler class + * identifies as those it owns. */ template static void count_dofs_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain, + const types::subdomain_id_t subdomain, std::vector &n_dofs_on_subdomain); + /** + * Return a set of indices that denotes + * the degrees of freedom that live on + * the given subdomain, i.e. that are on + * cells owned by the current + * processor. Note that this includes the + * ones that this subdomain "owns" + * (i.e. the ones for which + * get_subdomain_association() returns a + * value equal to the subdomain given + * here and that are selected by the + * extract_locally_owned() function) but + * also all of those that sit on the + * boundary between the given subdomain + * and other subdomain. In essence, + * degrees of freedom that sit on + * boundaries between subdomain will be + * in the index sets returned by this + * function for more than one subdomain. + * + * Note that this function is of + * questionable use for DoFHandler objects built on + * parallel::distributed::Triangulation + * since in that case ownership of + * individual degrees of freedom by MPI + * processes is controlled by the DoF + * handler object, not based on some + * geometric algorithm in conjunction + * with subdomain id. In particular, the + * degrees of freedom identified by the + * functions in this namespace as + * associated with a subdomain are not + * the same the + * DoFHandler class + * identifies as those it owns. + */ + template + static + IndexSet + dof_indices_with_subdomain_association (const DH &dof_handler, + const types::subdomain_id_t subdomain); + /** * Count how many degrees of * freedom out of the total @@ -1454,12 +1604,22 @@ class DoFTools * step-31, and * step-32 tutorial * programs. + * + * @pre The dofs_per_block + * variable has as many + * components as the finite + * element used by the + * dof_handler argument has + * blocks, or alternatively as + * many blocks as are enumerated + * in the target_blocks argument + * if given. */ template static void count_dofs_per_block (const DoFHandler& dof_handler, std::vector& dofs_per_block, - std::vector target_block + std::vector target_blocks = std::vector()); /** diff --git a/deal.II/deal.II/include/dofs/number_cache.h b/deal.II/deal.II/include/dofs/number_cache.h new file mode 100644 index 0000000000..e6bacba8f8 --- /dev/null +++ b/deal.II/deal.II/include/dofs/number_cache.h @@ -0,0 +1,101 @@ +//---------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2006, 2007, 2008, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//---------------------------------------------------------------------- +#ifndef __deal2__number_cache_h +#define __deal2__number_cache_h + +#include +#include + +#include + + +DEAL_II_NAMESPACE_OPEN + +namespace internal +{ + namespace DoFHandler + { + /** + * A structure used by the + * DoFHandler classes to store + * information about the degrees + * of freedom they deals with. + */ + struct NumberCache + { + NumberCache (); + + /** + * Determine an estimate for the + * memory consumption (in bytes) of + * this object. + */ + unsigned int memory_consumption () const; + + /** + * Total number of dofs, + * accumulated over all + * processors that may + * participate on this mesh. + */ + unsigned int n_global_dofs; + + /** + * Number of dofs owned by + * this MPI process. If this + * is a sequential + * computation, then this + * equals n_global_dofs. + */ + unsigned int n_locally_owned_dofs; + + /** + * An index set denoting the + * set of locally owned + * dofs. If this is a + * sequential computation, + * then it contains the + * entire range + * [0,n_global_dofs). + */ + IndexSet locally_owned_dofs; + + /** + * The number of dofs owned + * by each of the various MPI + * processes. If this is a + * sequential job, then the + * vector contains a single + * element equal to + * n_global_dofs. + */ + std::vector n_locally_owned_dofs_per_processor; + + /** + * The dofs owned by each of + * the various MPI + * processes. If this is a + * sequential job, then the + * vector has a single + * element equal to + * locally_owned_dofs. + */ + std::vector locally_owned_dofs_per_processor; + }; + } +} + + +DEAL_II_NAMESPACE_CLOSE + +#endif // __deal2__dof_iterator_selector_h diff --git a/deal.II/deal.II/include/grid/filtered_iterator.h b/deal.II/deal.II/include/grid/filtered_iterator.h index 0a7fb90297..c56fc0bc79 100644 --- a/deal.II/deal.II/include/grid/filtered_iterator.h +++ b/deal.II/deal.II/include/grid/filtered_iterator.h @@ -48,7 +48,7 @@ namespace IteratorFilters * * @ingroup Iterators */ - class Active + class Active { public: /** @@ -59,7 +59,7 @@ namespace IteratorFilters template bool operator () (const Iterator &i) const; }; - + /** * Filter that evaluates to true if * either the iterator points to an @@ -81,7 +81,7 @@ namespace IteratorFilters template bool operator () (const Iterator &i) const; }; - + /** * Filter that evaluates to true if @@ -106,7 +106,7 @@ namespace IteratorFilters bool operator () (const Iterator &i) const; }; - + /** * Filter for iterators that * evaluates to true if either the @@ -117,7 +117,7 @@ namespace IteratorFilters * * @ingroup Iterators */ - class LevelEqualTo + class LevelEqualTo { public: /** @@ -161,7 +161,7 @@ namespace IteratorFilters * * @ingroup Iterators */ - class SubdomainEqualTo + class SubdomainEqualTo { public: /** @@ -170,7 +170,7 @@ namespace IteratorFilters * shall have to be evaluated * to true. */ - SubdomainEqualTo (const unsigned int subdomain_id); + SubdomainEqualTo (const types::subdomain_id_t subdomain_id); /** * Evaluation operator. Returns @@ -188,7 +188,7 @@ namespace IteratorFilters * Stored value to compare the * subdomain with. */ - const unsigned int subdomain_id; + const types::subdomain_id_t subdomain_id; }; } @@ -245,7 +245,7 @@ namespace IteratorFilters * { * return (static_cast(c->level()) == level); * }; - * @endcode + * @endcode * then * @code * std::bind2nd (std::ptr_fun(&level_equal_to), 3) @@ -257,7 +257,7 @@ namespace IteratorFilters * * Finally, classes can be predicates. The following class is one: * @code - * class Active + * class Active * { * public: * template @@ -272,7 +272,7 @@ namespace IteratorFilters * class SubdomainEqualTo * { * public: - * SubdomainEqualTo (const unsigned int subdomain_id) + * SubdomainEqualTo (const types::subdomain_id_t subdomain_id) * : subdomain_id (subdomain_id) {}; * * template @@ -281,7 +281,7 @@ namespace IteratorFilters * } * * private: - * const unsigned int subdomain_id; + * const types::subdomain_id_t subdomain_id; * }; * @endcode * Objects like SubdomainEqualTo(3) can then be used as predicates. @@ -432,7 +432,7 @@ class FilteredIterator : public BaseIterator * will automatically be advanced * to the first cell that has. */ - template + template FilteredIterator (Predicate p, const BaseIterator &bi); @@ -447,7 +447,7 @@ class FilteredIterator : public BaseIterator * Destructor. */ ~FilteredIterator (); - + /** * Assignment operator. Copy the * iterator value of the @@ -490,7 +490,7 @@ class FilteredIterator : public BaseIterator */ FilteredIterator & set_to_next_positive (const BaseIterator &bi); - + /** * As above, but search for the * previous iterator from @p bi @@ -507,7 +507,7 @@ class FilteredIterator : public BaseIterator */ FilteredIterator & set_to_previous_positive (const BaseIterator &bi); - + /** * Compare for equality of the * underlying iterator values of @@ -559,7 +559,7 @@ class FilteredIterator : public BaseIterator * * We do not compare the * predicates. - */ + */ bool operator < (const FilteredIterator &fi) const; /** @@ -618,7 +618,7 @@ class FilteredIterator : public BaseIterator << "The element " << arg1 << " with which you want to compare or which you want to" << " assign from is invalid since it does not satisfy the predicate."); - + private: /** @@ -705,7 +705,7 @@ class FilteredIterator : public BaseIterator * type of this pointer. */ virtual PredicateBase * clone () const; - + private: /** * Copy of the predicate. @@ -830,7 +830,7 @@ FilteredIterator:: operator = (const BaseIterator &bi) { Assert ((bi.state() != IteratorState::valid) || (*predicate)(bi), - ExcInvalidElement(bi)); + ExcInvalidElement(bi)); BaseIterator::operator = (bi); return *this; } @@ -847,7 +847,7 @@ set_to_next_positive (const BaseIterator &bi) while ((this->state() == IteratorState::valid) && ( ! (*predicate)(*this))) BaseIterator::operator++ (); - + return *this; } @@ -863,7 +863,7 @@ set_to_previous_positive (const BaseIterator &bi) while ((this->state() == IteratorState::valid) && ( ! (*predicate)(*this))) BaseIterator::operator-- (); - + return *this; } @@ -964,7 +964,7 @@ FilteredIterator:: operator ++ (int) { const FilteredIterator old_state = *this; - + if (this->state() == IteratorState::valid) do BaseIterator::operator++ (); @@ -999,7 +999,7 @@ FilteredIterator:: operator -- (int) { const FilteredIterator old_state = *this; - + if (this->state() == IteratorState::valid) do BaseIterator::operator-- (); @@ -1043,7 +1043,7 @@ clone () const -namespace IteratorFilters +namespace IteratorFilters { // ---------------- IteratorFilters::Active --------- @@ -1079,7 +1079,7 @@ namespace IteratorFilters } -// ---------------- IteratorFilters::LevelEqualTo --------- +// ---------------- IteratorFilters::LevelEqualTo --------- inline LevelEqualTo::LevelEqualTo (const unsigned int level) : @@ -1100,7 +1100,7 @@ namespace IteratorFilters // ---------------- IteratorFilters::SubdomainEqualTo --------- inline - SubdomainEqualTo::SubdomainEqualTo (const unsigned int subdomain_id) + SubdomainEqualTo::SubdomainEqualTo (const types::subdomain_id_t subdomain_id) : subdomain_id (subdomain_id) {} @@ -1112,7 +1112,7 @@ namespace IteratorFilters bool SubdomainEqualTo::operator () (const Iterator &i) const { - return (static_cast(i->subdomain_id()) == subdomain_id); + return (i->subdomain_id() == subdomain_id); } } diff --git a/deal.II/deal.II/include/grid/grid_tools.h b/deal.II/deal.II/include/grid/grid_tools.h index c4ef0cc767..606709fc68 100644 --- a/deal.II/deal.II/include/grid/grid_tools.h +++ b/deal.II/deal.II/include/grid/grid_tools.h @@ -68,7 +68,7 @@ class GridTools * Same function, but for 1d. */ static - double diameter (const Triangulation<1> &tria); + double diameter (const Triangulation<1> &tria); /** * Same function, but for 1d, 2sd. */ @@ -95,7 +95,7 @@ class GridTools static double cell_measure (const std::vector > &all_vertices, const unsigned int (&vertex_indices)[GeometryInfo::vertices_per_cell]); - + /** * Remove vertices that are not * referenced by any of the @@ -124,7 +124,7 @@ class GridTools * by that class, so we have to * eliminate unused vertices * beforehand. - * + * * Not implemented for the * codimension one case. */ @@ -133,7 +133,7 @@ class GridTools void delete_unused_vertices (std::vector > &vertices, std::vector > &cells, SubCellData &subcelldata); - + /** * Remove vertices that are duplicated, * due to the input of a structured grid, @@ -164,7 +164,7 @@ class GridTools SubCellData &subcelldata, std::vector &considered_vertices, const double tol=1e-12); - + /** * Transform the vertices of the * given triangulation by @@ -362,7 +362,7 @@ class GridTools * the unit cell. In this case, * even points at a very small * distance outside the unit cell - * are allowed. + * are allowed. * * If a point lies on the * boundary of two or more cells, @@ -379,7 +379,7 @@ class GridTools * outside an actual unit cell, * due to numerical roundoff. * Therefore, the point returned - * by this function should + * by this function should * be projected onto the unit cell, * using GeometryInfo::project_to_unit_cell. * This is not automatically performed @@ -445,7 +445,7 @@ class GridTools template static void get_active_neighbors (const typename Container::active_cell_iterator &cell, - std::vector &active_neighbors); + std::vector &active_neighbors); /** * Produce a sparsity pattern in which @@ -462,7 +462,7 @@ class GridTools static void get_face_connectivity_of_cells (const Triangulation &triangulation, SparsityPattern &connectivity); - + /** * Use the METIS partitioner to generate * a partitioning of the active cells @@ -571,7 +571,7 @@ class GridTools partition_triangulation (const unsigned int n_partitions, const SparsityPattern &cell_connection_graph, Triangulation &triangulation); - + /** * For each active cell, return in the * output array to which subdomain (as @@ -590,16 +590,21 @@ class GridTools template static void get_subdomain_association (const Triangulation &triangulation, - std::vector &subdomain); + std::vector &subdomain); /** * Count how many cells are uniquely * associated with the given @p subdomain * index. * - * This function will generate an - * exception if there are no cells with - * the given @p subdomain index. + * This function may return zero + * if there are no cells with the + * given @p subdomain index. This + * can happen, for example, if + * you try to partition a coarse + * mesh into more partitions (one + * for each processor) than there + * are cells in the mesh. * * This function returns the number of * cells associated with one @@ -612,7 +617,7 @@ class GridTools template static unsigned int count_cells_with_subdomain_association (const Triangulation &triangulation, - const unsigned int subdomain); + const types::subdomain_id_t subdomain); /** * Given two mesh containers @@ -789,7 +794,7 @@ class GridTools typename Triangulation::DistortedCellList fix_up_distorted_child_cells (const typename Triangulation::DistortedCellList &distorted_cells, Triangulation &triangulation); - + /** * Exception */ @@ -849,7 +854,7 @@ void GridTools::transform (const Predicate &predicate, { Assert (triangulation.n_levels() == 1, ExcTriangulationHasBeenRefined()); - + std::vector treated_vertices (triangulation.n_vertices(), false); @@ -880,7 +885,7 @@ std::vector GridTools::get_active_child_cells (const typename DH::cell_iterator& cell) { std::vector child_cells; - + if (cell->has_children()) { for (unsigned int child=0; @@ -900,7 +905,7 @@ GridTools::get_active_child_cells (const typename DH::cell_iterator& cell) } - + #if deal_II_dimension == 1 template @@ -923,7 +928,7 @@ GridTools::get_active_neighbors(const typename Container::active_cell_iterator { while (neighbor_child->has_children()) neighbor_child = neighbor_child->child (n==0 ? 1 : 0); - + Assert (neighbor_child->neighbor(n==0 ? 1 : 0)==cell, ExcInternalError()); } diff --git a/deal.II/deal.II/include/grid/tria.h b/deal.II/deal.II/include/grid/tria.h index 893b6f63ed..0c634137c4 100644 --- a/deal.II/deal.II/include/grid/tria.h +++ b/deal.II/deal.II/include/grid/tria.h @@ -62,6 +62,7 @@ namespace hp } template class MGDoFHandler; + /*------------------------------------------------------------------------*/ /** @@ -1521,22 +1522,22 @@ class Triangulation : public Subscriptor * Reset this triangulation into a * virgin state by deleting all data. * - * Note that this operation is only allowed - * if no subscriptions to this object exist - * any more, such as DoFHandler objects - * using it. + * Note that this operation is only + * allowed if no subscriptions to this + * object exist any more, such as + * DoFHandler objects using it. */ - void clear (); + virtual void clear (); /** * Sets the mesh smoothing to @p * mesh_smoothing. This overrides * the MeshSmoothing given to the - * constructor. It is allow to + * constructor. It is allowed to * call this function only if - * triangulation is empty. + * the triangulation is empty. */ - void set_mesh_smoothing(const MeshSmoothing mesh_smoothing); + virtual void set_mesh_smoothing (const MeshSmoothing mesh_smoothing); /** * If @p dim==spacedim, assign a boundary @@ -1945,13 +1946,13 @@ class Triangulation : public Subscriptor * RefinementListener. Adding * listeners to the * Triangulation allows other - * classes to be informed, when + * classes to be informed when * the Triangulation is refined. */ void add_refinement_listener (RefinementListener &listener) const; /** - * Removes a + * Remove a * RefinementListener. When some * class needs no longer to be * informed about refinements, @@ -2295,8 +2296,9 @@ class Triangulation : public Subscriptor * Iterator to the first active * cell on level @p level. * - * This function calls @p begin_active_line - * in 1D and @p begin_active_quad in 2D. + * This function calls @p + * begin_active_line in 1D and @p + * begin_active_quad in 2D. */ active_cell_iterator begin_active(const unsigned int level = 0) const; @@ -2346,8 +2348,9 @@ class Triangulation : public Subscriptor raw_cell_iterator last_raw () const; /** - * Return an iterator pointing to the last - * cell of the level @p level, used or not. + * Return an iterator pointing to the + * last cell of the level @p level, used + * or not. * * This function calls @p last_raw_line * in 1D and @p last_raw_quad in 2D. @@ -2355,8 +2358,8 @@ class Triangulation : public Subscriptor raw_cell_iterator last_raw (const unsigned int level) const; /** - * Return an iterator pointing to the last - * used cell. + * Return an iterator pointing to the + * last used cell. * * This function calls @p last_line * in 1D and @p last_quad in 2D. @@ -2364,8 +2367,8 @@ class Triangulation : public Subscriptor cell_iterator last () const; /** - * Return an iterator pointing to the last - * used cell on level @p level. + * Return an iterator pointing to the + * last used cell on level @p level. * * This function calls @p last_line * in 1D and @p last_quad in 2D. @@ -2373,17 +2376,18 @@ class Triangulation : public Subscriptor cell_iterator last (const unsigned int level) const; /** - * Return an iterator pointing to the last - * active cell. + * Return an iterator pointing to the + * last active cell. * - * This function calls @p last_active_line - * in 1D and @p last_active_quad in 2D. + * This function calls @p + * last_active_line in 1D and @p + * last_active_quad in 2D. */ active_cell_iterator last_active () const; /** - * Return an iterator pointing to the last - * active cell on level @p level. + * Return an iterator pointing to the + * last active cell on level @p level. * * This function calls @p last_active_line * in 1D and @p last_active_quad in 2D. @@ -2420,8 +2424,9 @@ class Triangulation : public Subscriptor * Iterator to the first active * face. * - * This function calls @p begin_active_line - * in 2D and @p begin_active_quad in 3D. + * This function calls @p + * begin_active_line in 2D and @p + * begin_active_quad in 3D. */ active_face_iterator begin_active_face() const; @@ -2437,16 +2442,18 @@ class Triangulation : public Subscriptor raw_face_iterator end_face () const; /** - * Return a raw iterator which is past the end. - * This is the same as end() and is - * only for combatibility with older versions. + * Return a raw iterator which is past + * the end. This is the same as + * end() and is only for + * combatibility with older versions. */ raw_face_iterator end_raw_face () const; /** - * Return an active iterator which is past the end. - * This is the same as end() and is - * only for combatibility with older versions. + * Return an active iterator which is + * past the end. This is the same as + * end() and is only for + * combatibility with older versions. */ active_face_iterator end_active_face () const; @@ -2460,20 +2467,21 @@ class Triangulation : public Subscriptor raw_face_iterator last_raw_face () const; /** - * Return an iterator pointing to the last - * used face. + * Return an iterator pointing to the + * last used face. * - * This function calls @p last_line - * in 2D and @p last_quad in 3D. + * This function calls @p last_line in + * 2D and @p last_quad in 3D. */ face_iterator last_face () const; /** - * Return an iterator pointing to the last - * active face. + * Return an iterator pointing to the + * last active face. * - * This function calls @p last_active_line - * in 2D and @p last_active_quad in 3D. + * This function calls @p + * last_active_line in 2D and @p + * last_active_quad in 3D. */ active_face_iterator last_active_face () const; @@ -2487,8 +2495,8 @@ class Triangulation : public Subscriptor */ /*@{*/ /** - * Iterator to the first line, used - * or not, on level @p level. If a level + * Iterator to the first line, used or + * not, on level @p level. If a level * has no lines, a past-the-end iterator * is returned. * If lines are no cells, i.e. for @p dim>1 @@ -2531,18 +2539,18 @@ class Triangulation : public Subscriptor line_iterator end_line (const unsigned int level) const; /** - * Return a raw iterator which is the first - * iterator not on level. If @p level is - * the last level, then this returns - * end(). + * Return a raw iterator which is the + * first iterator not on level. If @p + * level is the last level, then this + * returns end(). */ raw_line_iterator end_raw_line (const unsigned int level) const; /** * Return an active iterator which is the - * first iterator not on level. If @p level - * is the last level, then this returns - * end(). + * first iterator not on level. If @p + * level is the last level, then this + * returns end(). */ active_line_iterator end_active_line (const unsigned int level) const; @@ -2554,36 +2562,37 @@ class Triangulation : public Subscriptor last_raw_line () const; /** - * Return an iterator pointing to the last - * line of the level @p level, used or not. + * Return an iterator pointing to the + * last line of the level @p level, used + * or not. */ raw_line_iterator last_raw_line (const unsigned int level) const; /** - * Return an iterator pointing to the last - * used line. + * Return an iterator pointing to the + * last used line. */ line_iterator last_line () const; /** - * Return an iterator pointing to the last - * used line on level @p level. + * Return an iterator pointing to the + * last used line on level @p level. */ line_iterator last_line (const unsigned int level) const; /** - * Return an iterator pointing to the last - * active line. + * Return an iterator pointing to the + * last active line. */ active_line_iterator last_active_line () const; /** - * Return an iterator pointing to the last - * active line on level @p level. + * Return an iterator pointing to the + * last active line on level @p level. */ active_line_iterator last_active_line (const unsigned int level) const; @@ -2597,12 +2606,12 @@ class Triangulation : public Subscriptor /*@{ */ /** - * Iterator to the first quad, used - * or not, on the given level. If a level + * Iterator to the first quad, used or + * not, on the given level. If a level * has no quads, a past-the-end iterator - * is returned. - * If quads are no cells, i.e. for $dim>2$ - * no level argument must be given. + * is returned. If quads are no cells, + * i.e. for $dim>2$ no level argument + * must be given. */ raw_quad_iterator @@ -2639,18 +2648,18 @@ class Triangulation : public Subscriptor quad_iterator end_quad (const unsigned int level) const; /** - * Return a raw iterator which is the first - * iterator not on level. If @p level is - * the last level, then this returns - * end(). + * Return a raw iterator which is the + * first iterator not on level. If @p + * level is the last level, then this + * returns end(). */ raw_quad_iterator end_raw_quad (const unsigned int level) const; /** * Return an active iterator which is the - * first iterator not on level. If @p level - * is the last level, then this returns - * end(). + * first iterator not on level. If @p + * level is the last level, then this + * returns end(). */ active_quad_iterator end_active_quad (const unsigned int level) const; @@ -2662,37 +2671,38 @@ class Triangulation : public Subscriptor last_raw_quad () const; /** - * Return an iterator pointing to the last - * quad of the level @p level, used or not. + * Return an iterator pointing to the + * last quad of the level @p level, used + * or not. */ raw_quad_iterator last_raw_quad (const unsigned int level) const; /** - * Return an iterator pointing to the last - * used quad. + * Return an iterator pointing to the + * last used quad. */ quad_iterator last_quad () const; /** - * Return an iterator pointing to the last - * used quad on level @p level. + * Return an iterator pointing to the + * last used quad on level @p level. */ quad_iterator last_quad (const unsigned int level) const; /** - * Return an iterator pointing to the last - * active quad. + * Return an iterator pointing to the + * last active quad. */ active_quad_iterator last_active_quad () const; /** - * Return an iterator pointing to the last - * active quad on level @p level. + * Return an iterator pointing to the + * last active quad on level @p level. */ active_quad_iterator last_active_quad (const unsigned int level) const; @@ -2755,9 +2765,9 @@ class Triangulation : public Subscriptor /** * Return an active iterator which is the - * first iterator not on level. If @p level - * is the last level, then this returns - * end(). + * first iterator not on level. If @p + * level is the last level, then this + * returns end(). */ active_hex_iterator end_active_hex (const unsigned int level) const; @@ -2769,37 +2779,38 @@ class Triangulation : public Subscriptor last_raw_hex () const; /** - * Return an iterator pointing to the last - * hex of the level @p level, used or not. + * Return an iterator pointing to the + * last hex of the level @p level, used + * or not. */ raw_hex_iterator last_raw_hex (const unsigned int level) const; /** - * Return an iterator pointing to the last - * used hex. + * Return an iterator pointing to the + * last used hex. */ hex_iterator last_hex () const; /** - * Return an iterator pointing to the last - * used hex on level @p level. + * Return an iterator pointing to the + * last used hex on level @p level. */ hex_iterator last_hex (const unsigned int level) const; /** - * Return an iterator pointing to the last - * active hex. + * Return an iterator pointing to the + * last active hex. */ active_hex_iterator last_active_hex () const; /** - * Return an iterator pointing to the last - * active hex on level @p level. + * Return an iterator pointing to the + * last active hex on level @p level. */ active_hex_iterator last_active_hex (const unsigned int level) const; @@ -2913,26 +2924,28 @@ class Triangulation : public Subscriptor unsigned int n_raw_hexs (const unsigned int level) const; /** - * Return total number of used hexahedra, - * active or not. + * Return total number of used + * hexahedra, active or not. */ unsigned int n_hexs() const; /** - * Return total number of used hexahedra, - * active or not on level @p level. + * Return total number of used + * hexahedra, active or not on level @p + * level. */ unsigned int n_hexs(const unsigned int level) const; /** - * Return total number of active hexahedra, - * active or not. + * Return total number of active + * hexahedra, active or not. */ unsigned int n_active_hexs() const; /** - * Return total number of active hexahedra, - * active or not on level @p level. + * Return total number of active + * hexahedra, active or not on level @p + * level. */ unsigned int n_active_hexs(const unsigned int level) const; @@ -2944,8 +2957,8 @@ class Triangulation : public Subscriptor /** * Return total number of used cells, - * active or not. - * Maps to n_lines() in one space + * active or not. Maps to + * n_lines() in one space * dimension and so on. */ unsigned int n_cells () const; @@ -2953,22 +2966,22 @@ class Triangulation : public Subscriptor /** * Return total number of used cells, * active or not, on level @p level. - * Maps to n_lines(level) in one space - * dimension and so on. + * Maps to n_lines(level) in + * one space dimension and so on. */ unsigned int n_cells (const unsigned int level) const; /** * Return total number of active cells. - * Maps to n_active_lines() in one space - * dimension and so on. + * Maps to n_active_lines() in + * one space dimension and so on. */ unsigned int n_active_cells () const; /** - * Return total number of active cells - * on level @p level. - * Maps to n_active_lines(level) in one + * Return total number of active cells on + * level @p level. Maps to + * n_active_lines(level) in one * space dimension and so on. */ unsigned int n_active_cells (const unsigned int level) const; @@ -3159,6 +3172,13 @@ class Triangulation : public Subscriptor << ", but this level is empty."); /*@}*/ protected: + /** + * Do some smoothing in the process + * of refining the triangulation. See + * the general doc of this class for + * more information about this. + */ + MeshSmoothing smooth_grid; /** * Write a bool vector to the given stream, @@ -3202,16 +3222,13 @@ class Triangulation : public Subscriptor std::istream &in); private: - /** - * The (public) clear() will only - * work when the triangulation is - * not subscriped to by other - * users. The - * clear_despite_subscriptions() - * function now allows the - * triangulation being cleared - * even when there are + * The (public) function clear() will + * only work when the triangulation is + * not subscribed to by other users. The + * clear_despite_subscriptions() function + * now allows the triangulation being + * cleared even when there are * subscriptions. * * Make sure, you know what you @@ -3327,14 +3344,7 @@ class Triangulation : public Subscriptor */ bool anisotropic_refinement; - /** - * Do some smoothing in the process - * of refining the triangulation. See - * the general doc of this class for - * more information about this. - */ - MeshSmoothing smooth_grid; - + /** * A flag that determines whether * we are to check for distorted diff --git a/deal.II/deal.II/include/grid/tria_accessor.h b/deal.II/deal.II/include/grid/tria_accessor.h index aa91f89254..9a60c48f80 100644 --- a/deal.II/deal.II/include/grid/tria_accessor.h +++ b/deal.II/deal.II/include/grid/tria_accessor.h @@ -70,7 +70,7 @@ namespace internal */ type () {} - + /** * Dummy * constructor. Only @@ -298,6 +298,7 @@ class TriaAccessorBase */ static const unsigned int structure_dimension = structdim; + protected: /** * Declare the data type that * this accessor class expects to @@ -708,7 +709,7 @@ class TriaAccessor : public TriaAccessorBase * returned. */ int parent_index () const; - + /** * @name Accessing sub-objects @@ -1317,7 +1318,7 @@ class TriaAccessor : public TriaAccessorBase */ double diameter () const; - /** + /** * Length of an object in the direction * of the given axis, specified in the * local coordinate system. See the @@ -1513,7 +1514,7 @@ class TriaAccessor : public TriaAccessorBase * in the library. */ void clear_refinement_case () const; - + /** * Set the parent of a cell. */ @@ -2187,22 +2188,54 @@ class CellAccessor : public TriaAccessor /** * Return the subdomain id of * this cell. + * + * See the @ref GlossSubdomainId + * "glossary" for more + * information. This function + * should not be called if you + * use a + * parallel::distributed::Triangulation + * object. */ - unsigned int subdomain_id () const; + types::subdomain_id_t subdomain_id () const; /** * Set the subdomain id of this * cell. + * + * See the @ref GlossSubdomainId + * "glossary" for more + * information. This function + * should not be called if you + * use a + * parallel::distributed::Triangulation + * object. + */ + void set_subdomain_id (const types::subdomain_id_t new_subdomain_id) const; + + /** + * Set the subdomain id of this + * cell and all its children (and + * grand-children, and so on) to + * the given value. + * + * See the @ref GlossSubdomainId + * "glossary" for more + * information. This function + * should not be called if you + * use a + * parallel::distributed::Triangulation + * object. */ - void set_subdomain_id (const unsigned int new_subdomain_id) const; - + void recursively_set_subdomain_id (const types::subdomain_id_t new_subdomain_id) const; + /** * Return an iterator to the * parent. */ TriaIterator > parent () const; - + /** * @} */ @@ -2218,9 +2251,71 @@ class CellAccessor : public TriaAccessor * Test whether the cell has children * (this is the criterion for activity * of a cell). + * + * See the @ref GlossActive "glossary" + * for more information. */ bool active () const; + /** + * Return whether this cell + * exists in the global mesh but + * (i) is owned by another + * processor, i.e. has a + * subdomain_id different from + * the one the current processor + * owns and (ii) is adjacent to a + * cell owned by the current + * processor. + * + * This function only makes sense + * if the triangulation used is + * of kind + * parallel::distributed::Triangulation. In + * all other cases, the returned + * value is always false. + * + * See the @ref GlossGhostCell + * "glossary" and the @ref + * distributed module for more + * information. + */ + bool is_ghost () const; + + /** + * Return whether this cell is + * artificial, i.e. it isn't one + * of the cells owned by the + * current processor, and it also + * doesn't border on one. As a + * consequence, it exists in the + * mesh to ensure that each + * processor has all coarse mesh + * cells and that the 2:1 ratio + * of neighboring cells is + * maintained, but it is not one + * of the cells we should work on + * on the current processor. In + * particular, there is no + * guarantee that this cell + * isn't, in fact, further + * refined on one of the other + * processors. + * + * This function only makes sense + * if the triangulation used is + * of kind + * parallel::distributed::Triangulation. In + * all other cases, the returned + * value is always false. + * + * See the @ref + * GlossArtificialCell "glossary" + * and the @ref distributed + * module for more information. + */ + bool is_artificial () const; + /** * Test whether the point @p p * is inside this cell. Points on diff --git a/deal.II/deal.II/include/grid/tria_accessor.templates.h b/deal.II/deal.II/include/grid/tria_accessor.templates.h index 69e83a4bc2..3665478044 100644 --- a/deal.II/deal.II/include/grid/tria_accessor.templates.h +++ b/deal.II/deal.II/include/grid/tria_accessor.templates.h @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -22,12 +23,13 @@ #include #include #include -#include +#include #include DEAL_II_NAMESPACE_OPEN + /*------------------------ Functions: TriaAccessorBase ---------------------------*/ template @@ -42,7 +44,7 @@ TriaAccessorBase::TriaAccessorBase ( present_index (index), tria (tria) { - + // non-cells have no level, so a 0 // should have been passed, or a -1 // for an end-iterator, or -2 for @@ -74,7 +76,7 @@ TriaAccessorBase::copy_from (const TriaAccessorBase::operator= (const TriaAccessorBase::operator -- () if (structdim != dim) { - if (this->present_index < 0) + if (this->present_index < 0) this->present_index = -1; } else { - while (this->present_index < 0) + while (this->present_index < 0) { // no -> go one level down --this->present_level; // lowest level reached? - if (this->present_level == -1) + if (this->present_level == -1) { // return with past the end pointer this->present_level = this->present_index = -1; @@ -278,7 +280,7 @@ namespace internal { return &faces->quads; } - + inline internal::Triangulation::TriaObjects >* get_objects (internal::Triangulation::TriaFaces<1>*, @@ -287,7 +289,7 @@ namespace internal Assert (false, ExcInternalError()); return 0; } - + inline internal::Triangulation::TriaObjects >* get_objects (internal::Triangulation::TriaFaces<2>*, @@ -296,7 +298,7 @@ namespace internal Assert (false, ExcInternalError()); return 0; } - + inline internal::Triangulation::TriaObjects >* get_objects (internal::Triangulation::TriaFaces<3>*, @@ -305,7 +307,7 @@ namespace internal Assert (false, ExcInternalError()); return 0; } - + /** * This function should never be * used, but we need it for the @@ -501,7 +503,7 @@ namespace internal // enclosing namespace // internal::TriaAccessor using dealii::TriaAccessor; - + /** * A class with the same purpose as the similarly named class of the * Triangulation class. See there for more information. @@ -536,7 +538,7 @@ namespace internal return accessor.objects().cells[accessor.present_index].face(i); } - + template static unsigned int @@ -572,7 +574,7 @@ namespace internal const unsigned int quad_index=lookup_table[i][0]; const unsigned int std_line_index=lookup_table[i][1]; - + const unsigned int line_index=GeometryInfo::standard_to_real_face_line( std_line_index, accessor.face_orientation(quad_index), @@ -602,7 +604,7 @@ namespace internal return numbers::invalid_unsigned_int; } - + template static unsigned int @@ -630,7 +632,7 @@ namespace internal { return true; } - + template static @@ -657,7 +659,7 @@ namespace internal { return true; } - + template static @@ -671,7 +673,7 @@ namespace internal < accessor.tria->levels[accessor.present_level] ->cells.face_flips.size(), ExcInternalError()); - + return (accessor.tria->levels[accessor.present_level] ->cells.face_flips[accessor.present_index * GeometryInfo<3>::faces_per_cell @@ -693,7 +695,7 @@ namespace internal { return true; } - + template static @@ -707,7 +709,7 @@ namespace internal < accessor.tria->levels[accessor.present_level] ->cells.face_rotations.size(), ExcInternalError()); - + return (accessor.tria->levels[accessor.present_level] ->cells.face_rotations[accessor.present_index * GeometryInfo<3>::faces_per_cell @@ -727,7 +729,7 @@ namespace internal { return true; } - + template static @@ -753,7 +755,7 @@ namespace internal //TODO: why is this face_orientation, not line_orientation as in the setter function? return accessor.tria->faces->quads.face_orientation(accessor.present_index, line); } - + template static @@ -791,13 +793,13 @@ namespace internal const unsigned int quad_index=lookup_table[line][0]; const unsigned int std_line_index=lookup_table[line][1]; - + const unsigned int line_index=GeometryInfo::standard_to_real_face_line( std_line_index, accessor.face_orientation(quad_index), accessor.face_flip(quad_index), accessor.face_rotation(quad_index)); - + // now we got to the correct line and ask // the quad for its line_orientation. however, if // the face is rotated, it might be possible, @@ -831,8 +833,8 @@ namespace internal { false, false }}, { { true, false }, { false, true }}}}; - - + + return (accessor.quad(quad_index) ->line_orientation(line_index) == bool_table[std_line_index/2] @@ -857,7 +859,7 @@ namespace internal { Assert (false, ExcInternalError()); } - + template static @@ -896,7 +898,7 @@ namespace internal { Assert (false, ExcInternalError()); } - + template static @@ -911,7 +913,7 @@ namespace internal < accessor.tria->levels[accessor.present_level] ->cells.face_flips.size(), ExcInternalError()); - + accessor.tria->levels[accessor.present_level] ->cells.face_flips[accessor.present_index * GeometryInfo<3>::faces_per_cell @@ -934,7 +936,7 @@ namespace internal { Assert (false, ExcInternalError()); } - + template static @@ -949,7 +951,7 @@ namespace internal < accessor.tria->levels[accessor.present_level] ->cells.face_rotations.size(), ExcInternalError()); - + accessor.tria->levels[accessor.present_level] ->cells.face_rotations[accessor.present_index * GeometryInfo<3>::faces_per_cell @@ -970,7 +972,7 @@ namespace internal { Assert (false, ExcInternalError()); } - + template static @@ -1006,7 +1008,7 @@ namespace internal + line] = value; } - + template static @@ -1049,9 +1051,9 @@ namespace internal // // second index: vertex index to be switched // (or not) - + static const unsigned int switch_table[2][2]={{1,0},{0,1}}; - + return accessor.line(corner%2) ->vertex_index(switch_table[accessor.line_orientation(corner%2)][corner/2]); } @@ -1104,7 +1106,7 @@ TriaAccessor (const Triangulation *parent, template inline bool -TriaAccessor::used () const +TriaAccessor::used () const { Assert (this->state() == IteratorState::valid, TriaAccessorExceptions::ExcDereferenceInvalidObject()); @@ -1120,7 +1122,7 @@ vertex_index (const unsigned int corner) const { Assert (corner::vertices_per_cell, ExcIndexRange(corner,0,GeometryInfo::vertices_per_cell)); - + return internal::TriaAccessor::Implementation::vertex_index (*this, corner); } @@ -1189,7 +1191,7 @@ bool TriaAccessor::face_orientation (const unsigned int face) const { Assert (used(), TriaAccessorExceptions::ExcCellNotUsed()); - + return internal::TriaAccessor::Implementation::face_orientation (*this, face); } @@ -1201,7 +1203,7 @@ bool TriaAccessor::face_flip (const unsigned int face) const { Assert (used(), TriaAccessorExceptions::ExcCellNotUsed()); - + return internal::TriaAccessor::Implementation::face_flip (*this, face); } @@ -1239,7 +1241,7 @@ TriaAccessor::set_face_orientation (const unsigned int f const bool value) const { Assert (used(), TriaAccessorExceptions::ExcCellNotUsed()); - + internal::TriaAccessor::Implementation::set_face_orientation (*this, face, value); } @@ -1252,7 +1254,7 @@ TriaAccessor::set_face_flip (const unsigned int face, const bool value) const { Assert (used(), TriaAccessorExceptions::ExcCellNotUsed()); - + internal::TriaAccessor::Implementation::set_face_flip (*this, face, value); } @@ -1327,7 +1329,7 @@ child_index (const unsigned int i) const Assert (has_children(), TriaAccessorExceptions::ExcCellHasNoChildren()); Assert (i::no_refinement, TriaAccessorExceptions::ExcCellHasNoChildren()); - + if (this_refinement_case == RefinementCase<2>::cut_xy) return child_index(i); else if ((this_refinement_case == RefinementCase<2>::cut_x) @@ -1462,7 +1464,7 @@ isotropic_child (const unsigned int i) const Assert (this_refinement_case != RefinementCase<2>::no_refinement, TriaAccessorExceptions::ExcCellHasNoChildren()); - + if (this_refinement_case == RefinementCase<2>::cut_xy) return child(i); else if ((this_refinement_case == RefinementCase<2>::cut_x) @@ -1475,7 +1477,7 @@ isotropic_child (const unsigned int i) const return child(i/2)->child(i%2); else Assert(false, - ExcMessage("This cell has no grandchildren equivalent to isotropic refinement")); + ExcMessage("This cell has no grandchildren equivalent to isotropic refinement")); } default: @@ -1564,14 +1566,14 @@ TriaAccessor::set_children (const unsigned int i, // each set of two children are stored // consecutively, so we only have to find // the location of the set of children - const unsigned int n_sets_of_two = GeometryInfo::max_children_per_cell/2; + const unsigned int n_sets_of_two = GeometryInfo::max_children_per_cell/2; Assert ((index==-1) || (i==0 && !this->has_children() && (index>=0)) || (i>0 && this->has_children() && (index>=0) && this->objects().children[n_sets_of_two*this->present_index+i/2] == -1), TriaAccessorExceptions::ExcCantSetChildren(index)); - + this->objects().children[n_sets_of_two*this->present_index+i/2] = index; } @@ -1618,7 +1620,7 @@ TriaAccessor::user_flag_set () const template inline void -TriaAccessor::set_user_flag () const +TriaAccessor::set_user_flag () const { Assert (this->used(), TriaAccessorExceptions::ExcCellNotUsed()); this->objects().user_flags[this->present_index] = true; @@ -1634,7 +1636,7 @@ TriaAccessor::clear_user_flag () const Assert (this->used(), TriaAccessorExceptions::ExcCellNotUsed()); this->objects().user_flags[this->present_index] = false; } - + template @@ -1988,7 +1990,7 @@ CellAccessor (const CellAccessor &cell_accessor) template inline TriaIterator > -CellAccessor::face (const unsigned int i) const +CellAccessor::face (const unsigned int i) const { switch (dim) { @@ -2014,7 +2016,7 @@ CellAccessor::face (const unsigned int i) const template inline unsigned int -CellAccessor::face_index (const unsigned int i) const +CellAccessor::face_index (const unsigned int i) const { switch (dim) { @@ -2026,10 +2028,10 @@ CellAccessor::face_index (const unsigned int i) const case 2: return this->line_index(i); - + case 3: return this->quad_index(i); - + default: return numbers::invalid_unsigned_int; } @@ -2040,7 +2042,7 @@ CellAccessor::face_index (const unsigned int i) const template inline int -CellAccessor::neighbor_index (const unsigned int i) const +CellAccessor::neighbor_index (const unsigned int i) const { AssertIndexRange (i,GeometryInfo::faces_per_cell); return this->tria->levels[this->present_level]-> @@ -2087,7 +2089,7 @@ CellAccessor::set_refine_flag (const RefinementCase refinemen Assert (this->used() && this->active(), ExcRefineCellNotActive()); Assert (!coarsen_flag_set(), ExcCellFlaggedForCoarsening()); - + this->tria->levels[this->present_level]->refine_flags[this->present_index] = refinement_case; } @@ -2116,13 +2118,13 @@ CellAccessor::flag_for_face_refinement (const unsigned int face_no ExcIndexRange(face_no,0,GeometryInfo::faces_per_cell)); Assert (face_refinement_case < RefinementCase::isotropic_refinement+1, ExcIndexRange(face_refinement_case,0,RefinementCase::isotropic_refinement+1)); - + // the new refinement case is a combination // of the minimum required one for the given // face refinement and the already existing // flagged refinement case RefinementCase old_ref_case = refine_flag_set(); - RefinementCase + RefinementCase new_ref_case = (old_ref_case | GeometryInfo::min_cell_refinement_case_for_face_refinement(face_refinement_case, face_no, @@ -2225,7 +2227,7 @@ CellAccessor<3>::subface_case(const unsigned int face_no) const { Assert(face(face_no)->child(1)->refinement_case()==RefinementCase<2>::cut_y, ExcInternalError()); - return internal::SubfaceCase<3>::case_x1y2y; + return internal::SubfaceCase<3>::case_x1y2y; } else return internal::SubfaceCase<3>::case_x1y; @@ -2251,7 +2253,7 @@ CellAccessor<3>::subface_case(const unsigned int face_no) const { Assert(face(face_no)->child(1)->refinement_case()==RefinementCase<2>::cut_x, ExcInternalError()); - return internal::SubfaceCase<3>::case_y1x2x; + return internal::SubfaceCase<3>::case_y1x2x; } else return internal::SubfaceCase<3>::case_y1x; @@ -2305,7 +2307,7 @@ CellAccessor::set_coarsen_flag () const { Assert (this->used() && this->active(), ExcRefineCellNotActive()); Assert (!refine_flag_set(), ExcCellFlaggedForRefinement()); - + this->tria->levels[this->present_level]->coarsen_flags[this->present_index] = true; } @@ -2364,6 +2366,44 @@ CellAccessor::active () const +template +inline +bool +CellAccessor::is_ghost () const +{ +#ifndef DEAL_II_USE_P4EST + return false; +#else + const types::subdomain_id_t subdomain = this->subdomain_id(); + if (subdomain == types::artificial_subdomain_id) + return false; + + const parallel::distributed::Triangulation *pdt + = dynamic_cast *>(this->tria); + + if (pdt == 0) + return false; + else + return (subdomain != pdt->locally_owned_subdomain()); +#endif +} + + + +template +inline +bool +CellAccessor::is_artificial () const +{ +#ifndef DEAL_II_USE_P4EST + return false; +#else + return (this->subdomain_id() == types::artificial_subdomain_id); +#endif +} + + + template inline unsigned int @@ -2378,7 +2418,7 @@ CellAccessor::neighbor_face_no (const unsigned int neighbor) const // return this value as the // neighbor is not coarser return n2; - else + else // the neighbor is coarser return neighbor_of_coarser_neighbor(neighbor).first; } diff --git a/deal.II/deal.II/include/grid/tria_levels.h b/deal.II/deal.II/include/grid/tria_levels.h index 9974135839..6f2f0dbd06 100644 --- a/deal.II/deal.II/include/grid/tria_levels.h +++ b/deal.II/deal.II/include/grid/tria_levels.h @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2010 by the deal.II authors +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -75,7 +75,7 @@ namespace internal * coarsened. */ std::vector coarsen_flags; - + /** * Levels and indices of the neighbors * of the cells. Convention is, that the @@ -137,15 +137,15 @@ namespace internal * cells with a given subdomain * number. */ - std::vector subdomain_ids; - + std::vector subdomain_ids; + /** * One integer for every consecutive * pair of cells to store which * index their parent has. */ std::vector parents; - + /** * Reserve enough space to accomodate * @p total_cells cells on this level. @@ -180,7 +180,7 @@ namespace internal * of this object. */ unsigned int memory_consumption () const; - + /** * The object containing the data on lines and * related functions @@ -204,9 +204,9 @@ namespace internal << "The containers have sizes " << arg1 << " and " << arg2 << ", which is not as expected."); }; - + //TODO: Replace TriaObjectsHex to avoid this specialization - + /** * Specialization of TriaLevels for 3D. Since we need TriaObjectsHex * instead of TriaObjects. Refer to the documentation of the template @@ -220,7 +220,7 @@ namespace internal std::vector coarsen_flags; std::vector > neighbors; std::vector parents; - std::vector subdomain_ids; + std::vector subdomain_ids; void reserve_space (const unsigned int total_cells, const unsigned int dimension); void monitor_memory (const unsigned int true_dimension) const; @@ -244,8 +244,8 @@ namespace internal << "The containers have sizes " << arg1 << " and " << arg2 << ", which is not as expected."); }; - - + + } } diff --git a/deal.II/deal.II/include/hp/dof_handler.h b/deal.II/deal.II/include/hp/dof_handler.h index b45ac89836..b4264ae56e 100644 --- a/deal.II/deal.II/include/hp/dof_handler.h +++ b/deal.II/deal.II/include/hp/dof_handler.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -800,12 +801,37 @@ namespace hp /*---------------------------------------*/ - /** - * Return number of degrees of freedom. - * Included in this number are those - * DoFs which are constrained by - * hanging nodes. - */ + /** + * Return the global number of + * degrees of freedom. If the + * current object handles all + * degrees of freedom itself + * (even if you may intend to + * solve your linear system in + * parallel, such as in step-17 + * or step-18), then this number + * equals the number of locally + * owned degrees of freedom since + * this object doesn't know + * anything about what you want + * to do with it and believes + * that it owns every degree of + * freedom it knows about. + * + * On the other hand, if this + * object operates on a + * parallel::distributed::Triangulation + * object, then this function + * returns the global number of + * degrees of freedom, + * accumulated over all + * processors. + * + * In either case, included in + * the returned number are those + * DoFs which are constrained by + * hanging nodes, see @ref constraints. + */ unsigned int n_dofs () const; /** @@ -841,6 +867,97 @@ namespace hp unsigned int n_boundary_dofs (const std::set &boundary_indicators) const; + /** + * Return the number of + * degrees of freedom that + * belong to this + * process. + * + * If this is a sequential job, + * then the result equals that + * produced by n_dofs(). On the + * other hand, if we are + * operating on a + * parallel::distributed::Triangulation, + * then it includes only the + * degrees of freedom that the + * current processor owns. Note + * that in this case this does + * not include all degrees of + * freedom that have been + * distributed on the current + * processor's image of the mesh: + * in particular, some of the + * degrees of freedom on the + * interface between the cells + * owned by this processor and + * cells owned by other + * processors may be theirs, and + * degrees of freedom on ghost + * cells are also not necessarily + * included. + */ + unsigned int n_locally_owned_dofs() const; + + /** + * Return an IndexSet describing + * the set of locally owned DoFs + * as a subset of + * 0..n_dofs(). The number of + * elements of this set equals + * n_locally_owned_dofs(). + */ + const IndexSet & locally_owned_dofs() const; + + + /** + * Returns a vector that + * stores the locally owned + * DoFs of each processor. If + * you are only interested in + * the number of elements + * each processor owns then + * n_dofs_per_processor() is + * a better choice. + * + * If this is a sequential job, + * then the vector has a single + * element that equals the + * IndexSet representing the + * entire range [0,n_dofs()]. + */ + const std::vector & + locally_owned_dofs_per_processor () const; + + /** + * Return a vector that + * stores the number of + * degrees of freedom each + * processor that + * participates in this + * triangulation owns + * locally. The sum of all + * these numbers equals the + * number of degrees of + * freedom that exist + * globally, i.e. what + * n_dofs() returns. + * + * Each element of the vector + * returned by this function + * equals the number of + * elements of the + * corresponding sets + * returned by + * global_dof_indices(). + * + * If this is a sequential job, + * then the vector has a single + * element equal to n_dofs(). + */ + const std::vector & + n_locally_owned_dofs_per_processor () const; + /** * Return a constant reference to * the set of finite element @@ -1100,12 +1217,18 @@ namespace hp */ internal::hp::DoFFaces * faces; - - /** - * Store the number of dofs - * created last time. - */ - unsigned int used_dofs; + /** + * A structure that contains all + * sorts of numbers that + * characterize the degrees of + * freedom this object works on. + * + * For most members of this + * structure, there is an + * accessor function in this + * class that returns its value. + */ + internal::DoFHandler::NumberCache number_cache; /** * Array to store the indices @@ -1200,15 +1323,44 @@ namespace hp /* ----------------------- Inline functions ---------------------------------- */ - template + template inline unsigned int DoFHandler::n_dofs () const { - Assert (finite_elements != 0, - ExcMessage ("No finite element collection is associated with " - "this DoFHandler")); - return used_dofs; + return number_cache.n_global_dofs; + } + + + template + unsigned int + DoFHandler::n_locally_owned_dofs() const + { + return number_cache.n_locally_owned_dofs; + } + + + template + const IndexSet & + DoFHandler::locally_owned_dofs() const + { + return number_cache.locally_owned_dofs; + } + + + template + const std::vector & + DoFHandler::n_locally_owned_dofs_per_processor() const + { + return number_cache.n_locally_owned_dofs_per_processor; + } + + + template + const std::vector & + DoFHandler::locally_owned_dofs_per_processor () const + { + return number_cache.locally_owned_dofs_per_processor; } diff --git a/deal.II/deal.II/include/numerics/data_out.h b/deal.II/deal.II/include/numerics/data_out.h index dffd98ea68..c0e627abbd 100644 --- a/deal.II/deal.II/include/numerics/data_out.h +++ b/deal.II/deal.II/include/numerics/data_out.h @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors +// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -79,7 +79,7 @@ namespace internal const unsigned int n_q_points, const std::vector &n_postprocessor_outputs, const FE &finite_elements); - + const unsigned int n_components; const unsigned int n_datasets; const unsigned int n_subdivisions; @@ -92,7 +92,7 @@ namespace internal std::vector > > patch_hessians_system; std::vector > > postprocessed_values; - const dealii::hp::FECollection fe_collection; + const dealii::hp::FECollection fe_collection; }; @@ -223,7 +223,7 @@ namespace internal * if () ... else ... clauses yourself), and also functions * and classes offering ways to control the appearance of the output * by setting flags for each output format. - * + * * *

Information for derived classes

* @@ -316,19 +316,19 @@ class DataOut_DoFData : public DataOutInterface * associated to degrees of freedom */ type_dof_data, - + /** * Data vector entries are one per * grid cell */ type_cell_data, - + /** * Find out automatically */ type_automatic }; - + /** * Constructor */ @@ -620,7 +620,7 @@ class DataOut_DoFData : public DataOutInterface template void merge_patches (const DataOut_DoFData &source, const Point &shift = Point()); - + /** * Release the pointers to the * data vectors and the DoF @@ -706,7 +706,7 @@ class DataOut_DoFData : public DataOutInterface << "components. However, the vector component at\n" << "position " << arg1 << " with name <" << arg2 << "> does not satisfy these conditions."); - + protected: /** * For each vector that has been added @@ -753,12 +753,12 @@ class DataOut_DoFData : public DataOutInterface * be aquired from the postprocessor. */ DataEntryBase (const DataPostprocessor *data_postprocessor); - + /** * Destructor made virtual. */ virtual ~DataEntryBase (); - + /** * Assuming that the stored vector is * a cell vector, extract the given @@ -778,7 +778,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_values (const FEValuesBase &fe_patch_values, std::vector &patch_values) const = 0; - + /** * Given a FEValuesBase object, * extract the values on the present @@ -802,7 +802,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_gradients (const FEValuesBase &fe_patch_values, std::vector > &patch_gradients) const = 0; - + /** * Given a FEValuesBase object, * extract the gradients on the present @@ -826,7 +826,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_hessians (const FEValuesBase &fe_patch_values, std::vector > &patch_hessians) const = 0; - + /** * Given a FEValuesBase object, extract * the second derivatives on the @@ -845,7 +845,7 @@ class DataOut_DoFData : public DataOutInterface * vectors. */ virtual void clear () = 0; - + /** * Determine an estimate for * the memory consumption (in @@ -869,7 +869,7 @@ class DataOut_DoFData : public DataOutInterface */ const std::vector data_component_interpretation; - + /** * Pointer to a DataPostprocessing * object which shall be applied to @@ -945,7 +945,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_values (const FEValuesBase &fe_patch_values, std::vector &patch_values) const; - + /** * Given a FEValuesBase object, * extract the values on the present @@ -969,7 +969,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_gradients (const FEValuesBase &fe_patch_values, std::vector > &patch_gradients) const; - + /** * Given a FEValuesBase object, * extract the gradients on the present @@ -993,7 +993,7 @@ class DataOut_DoFData : public DataOutInterface void get_function_hessians (const FEValuesBase &fe_patch_values, std::vector > &patch_hessians) const; - + /** * Given a FEValuesBase object, extract * the second derivatives on the @@ -1012,7 +1012,7 @@ class DataOut_DoFData : public DataOutInterface * vectors. */ virtual void clear (); - + /** * Determine an estimate for * the memory consumption (in @@ -1063,7 +1063,7 @@ class DataOut_DoFData : public DataOutInterface * classes. */ std::vector patches; - + /** * Function by which the base * class's functions get to know @@ -1091,7 +1091,7 @@ class DataOut_DoFData : public DataOutInterface virtual std::vector > get_vector_data_ranges () const; - + /** * Make all template siblings * friends. Needed for the @@ -1139,20 +1139,21 @@ class DataOut_DoFData : public DataOutInterface * * The only thing this class offers is the function build_patches() which * loops over all cells of the triangulation stored by the - * attach_dof_handler() function of the base class and convert the data on - * these to actual patches which are the objects that are later output by the - * functions of the base classes. You can give a parameter to the function - * which determines how many subdivisions in each coordinate direction are to - * be performed, i.e. of how many subcells each patch shall consist. Default - * is one, but you may want to choose a higher number for higher order - * elements, so as two for quadratic elements, three for cubic elements three, - * and so on. The purpose of this parameter is because most graphics programs - * do not allow to specify higher order shape functions in the file formats: - * only data at vertices can be plotted and is then shown as a bilinear - * interpolation within the interior of cells. This may be insufficient if you - * have higher order finite elements, and the only way to achieve better - * output is to subdivide each cell of the mesh into several cells for - * graphical output. + * attach_dof_handler() function of the base class (with the exception of + * cells of parallel::distributed::Triangulation objects that are not owned by + * the current processor) and converts the data on these to actual patches + * which are the objects that are later output by the functions of the base + * classes. You can give a parameter to the function which determines how many + * subdivisions in each coordinate direction are to be performed, i.e. of how + * many subcells each patch shall consist. Default is one, but you may want to + * choose a higher number for higher order elements, so as two for quadratic + * elements, three for cubic elements three, and so on. The purpose of this + * parameter is because most graphics programs do not allow to specify higher + * order shape functions in the file formats: only data at vertices can be + * plotted and is then shown as a bilinear interpolation within the interior + * of cells. This may be insufficient if you have higher order finite + * elements, and the only way to achieve better output is to subdivide each + * cell of the mesh into several cells for graphical output. * * Note that after having called build_patches() once, you can call one or * more of the write() functions of DataOutInterface. You can therefore @@ -1200,12 +1201,12 @@ class DataOut_DoFData : public DataOutInterface * cannot be interpolated to a coarser cell. If you do have cell data and use * this pair of functions and they return a non-active cell, then an exception * will be thrown. - * + * * @ingroup output * @author Wolfgang Bangerth, 1999 */ template > -class DataOut : public DataOut_DoFData +class DataOut : public DataOut_DoFData { public: /** @@ -1227,7 +1228,7 @@ class DataOut : public DataOut_DoFData curved_boundary, curved_inner_cells }; - + /** * This is the central function * of this class since it builds @@ -1300,7 +1301,7 @@ class DataOut : public DataOut_DoFData virtual void build_patches (const Mapping &mapping, const unsigned int n_subdivisions = 0, const CurvedCellRegion curved_region = curved_boundary); - + /** * Return the first cell which we * want output for. The default @@ -1310,7 +1311,7 @@ class DataOut : public DataOut_DoFData * cells in a derived class. */ virtual cell_iterator first_cell (); - + /** * Return the next cell after * @p cell which we want output @@ -1341,12 +1342,40 @@ class DataOut : public DataOut_DoFData int, << "The number of subdivisions per patch, " << arg1 << ", is not valid."); - + private: + + /** + * Return the first cell produced + * by the + * first_cell()/next_cell() + * function pair that is locally + * owned. If this object operates + * on a non-distributed + * triangulation, the result + * equals what first_cell() + * returns. + */ + cell_iterator first_locally_owned_cell (); + + /** + * Return the next cell produced + * by the next_cell() function + * that is locally owned. If this + * object operates on a + * non-distributed triangulation, + * the result equals what + * first_cell() returns. + */ + cell_iterator next_locally_owned_cell (const cell_iterator &cell); + /** * Build one patch. This function * is called in a WorkStream * context. + * + * The result is written into the patch + * variable. */ void build_one_patch (const std::pair *cell_and_index, internal::DataOut::ParallelData &data, @@ -1448,7 +1477,7 @@ merge_patches (const DataOut_DoFData &source, ExcMessage ("Both sources need to declare the same components " "as vectors.")); } - + // merge patches. store old number // of elements, since we need to // adjust patch numbers, etc @@ -1463,11 +1492,11 @@ merge_patches (const DataOut_DoFData &source, for (unsigned int i=old_n_patches; i::vertices_per_cell; ++v) patches[i].vertices[v] += shift; - + // adjust patch numbers for (unsigned int i=old_n_patches; i::faces_per_cell; ++n) diff --git a/deal.II/deal.II/include/numerics/error_estimator.h b/deal.II/deal.II/include/numerics/error_estimator.h index 46ae47d84f..9a8bd60395 100644 --- a/deal.II/deal.II/include/numerics/error_estimator.h +++ b/deal.II/deal.II/include/numerics/error_estimator.h @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008 by the deal.II authors +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -61,7 +61,7 @@ namespace hp * since accuracy is not so important here, and since this can save rather * a lot of memory, when using many cells. * - * + * *

Implementation

* * In principle, the implementation of the error estimation is simple: let @@ -102,7 +102,7 @@ namespace hp * $h$ is taken to be the greatest length of the diagonals of the cell. For * more or less uniform cells without deformed angles, this coincides with * the diameter of the cell. - * + * * *

Vector-valued functions

* @@ -127,7 +127,7 @@ namespace hp * * *

%Boundary values

- * + * * If the face is at the boundary, i.e. there is no neighboring cell to which * the jump in the gradiend could be computed, there are two possibilities: *
    @@ -163,9 +163,9 @@ namespace hp * Thanks go to Franz-Theo Suttmeier for clarifications about boundary * conditions. * - * + * *

    Handling of hanging nodes

    - * + * * The integration along faces with hanging nodes is quite tricky, since one * of the elements has to be shifted one level up or down. See the * documentation for the FESubFaceValues class for more information about @@ -272,7 +272,7 @@ class KellyErrorEstimator * afterwards. This is in particular true * for the case where meshes are very * large and computing indicators for @em - * every cells is too expensive, while + * every cell is too expensive, while * computing indicators for only local * cells is acceptable. Note that if you * only ask for the indicators of a @@ -301,6 +301,29 @@ class KellyErrorEstimator * parameter is retained for * compatibility with old versions of the * library. + * + * @note If the DoFHandler object + * given as an argument to this + * function builds on a + * parallel::distributed::Triangulation, + * this function skips + * computations on all cells that + * are not locally owned. In that + * case, the only valid value for + * the subdomain_id argument + * (besides the invalid value) is + * the subdomain id that is + * associated with the currently + * processor, as reported by + * parallel::distributed::Triangulation::locally_owned_subdomain(). Even + * though nothing is computed on + * cells that we don't locally + * own, the error indicator + * vector must still have a + * length equal to the number of + * active cell in the mesh as + * reported by + * parallel::distributed::Triangulation::n_locally_owned_active_cells(). */ template static void estimate (const Mapping &mapping, @@ -311,15 +334,15 @@ class KellyErrorEstimator Vector &error, const std::vector &component_mask = std::vector(), const Function *coefficients = 0, - const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const unsigned int n_threads = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); - + /** * Calls the @p estimate * function, see above, with * mapping=MappingQ1@(). - */ + */ template static void estimate (const DH &dof, const Quadrature &quadrature, @@ -329,9 +352,9 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); - + /** * Same function as above, but * accepts more than one solution @@ -369,7 +392,7 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); /** @@ -386,7 +409,7 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -406,7 +429,7 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -425,7 +448,7 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -445,7 +468,7 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -464,10 +487,10 @@ class KellyErrorEstimator const std::vector &component_mask = std::vector(), const Function *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); - - + + /** * Exception */ @@ -575,14 +598,14 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); /** * Calls the @p estimate * function, see above, with * mapping=MappingQ1<1>(). - */ + */ template static void estimate (const DH &dof, const Quadrature<0> &quadrature, @@ -592,9 +615,9 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); - + /** * Same function as above, but * accepts more than one solution @@ -632,7 +655,7 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); /** @@ -649,7 +672,7 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -669,7 +692,7 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -688,7 +711,7 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -708,7 +731,7 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); @@ -727,9 +750,9 @@ class KellyErrorEstimator<1,spacedim> const std::vector &component_mask = std::vector(), const Function<1> *coefficients = 0, const unsigned int n_threads = multithread_info.n_default_threads, - const unsigned int subdomain_id = numbers::invalid_unsigned_int, + const types::subdomain_id_t subdomain_id = types::invalid_subdomain_id, const unsigned int material_id = numbers::invalid_unsigned_int); - + /** * Exception */ diff --git a/deal.II/deal.II/include/numerics/vectors.templates.h b/deal.II/deal.II/include/numerics/vectors.templates.h index 842b437a44..e0494237be 100644 --- a/deal.II/deal.II/include/numerics/vectors.templates.h +++ b/deal.II/deal.II/include/numerics/vectors.templates.h @@ -186,6 +186,7 @@ void VectorTools::interpolate (const Mapping fe, support_quadrature, update_quadrature_points); for (; cell!=endc; ++cell) + if (!cell->is_artificial() && !cell->is_ghost()) { const unsigned int fe_index = cell->active_fe_index(); @@ -1237,9 +1238,9 @@ VectorTools::interpolate_boundary_values (const Mapping &component_mask_) { //ConstraintMatrix boundary_constraints(); - // interpolate_boundary_values (dof, boundary_component, boundary_function, + // interpolate_boundary_values (dof, boundary_component, boundary_function, // boundary_constraints, component_mask); - + const unsigned int dim=DH::dimension; const unsigned int spacedim=DH::space_dimension; @@ -1489,197 +1490,198 @@ interpolate_boundary_values (const Mapping typename DH::active_cell_iterator cell = dof.begin_active(), endc = dof.end(); for (; cell!=endc; ++cell) - for (unsigned int face_no = 0; face_no < GeometryInfo::faces_per_cell; - ++face_no) - { - const FiniteElement &fe = cell->get_fe(); - - // we can presently deal only with - // primitive elements for boundary - // values. this does not preclude - // us using non-primitive elements - // in components that we aren't - // interested in, however. make - // sure that all shape functions - // that are non-zero for the - // components we are interested in, - // are in fact primitive - for (unsigned int i=0; iget_fe().dofs_per_cell; ++i) - { - const std::vector &nonzero_component_array - = cell->get_fe().get_nonzero_components (i); - for (unsigned int c=0; cget_fe().is_primitive (i), - ExcMessage ("This function can only deal with requested boundary " - "values that correspond to primitive (scalar) base " - "elements")); - } + if (!cell->is_artificial()) + for (unsigned int face_no = 0; face_no < GeometryInfo::faces_per_cell; + ++face_no) + { + const FiniteElement &fe = cell->get_fe(); + + // we can presently deal only with + // primitive elements for boundary + // values. this does not preclude + // us using non-primitive elements + // in components that we aren't + // interested in, however. make + // sure that all shape functions + // that are non-zero for the + // components we are interested in, + // are in fact primitive + for (unsigned int i=0; iget_fe().dofs_per_cell; ++i) + { + const std::vector &nonzero_component_array + = cell->get_fe().get_nonzero_components (i); + for (unsigned int c=0; cget_fe().is_primitive (i), + ExcMessage ("This function can only deal with requested boundary " + "values that correspond to primitive (scalar) base " + "elements")); + } - typename DH::face_iterator face = cell->face(face_no); - const unsigned char boundary_component = face->boundary_indicator(); - if (function_map.find(boundary_component) != function_map.end()) - { - // face is of the right component - x_fe_values.reinit(cell, face_no); - const FEFaceValues &fe_values = x_fe_values.get_present_fe_values(); - - // get indices, physical location and - // boundary values of dofs on this - // face - face_dofs.resize (fe.dofs_per_face); - face->get_dof_indices (face_dofs, cell->active_fe_index()); - const std::vector > &dof_locations - = fe_values.get_quadrature_points (); + typename DH::face_iterator face = cell->face(face_no); + const unsigned char boundary_component = face->boundary_indicator(); + if (function_map.find(boundary_component) != function_map.end()) + { + // face is of the right component + x_fe_values.reinit(cell, face_no); + const FEFaceValues &fe_values = x_fe_values.get_present_fe_values(); - if (fe_is_system) - { - // resize - // array. avoid - // construction of a - // memory allocating - // temporary if - // possible - if (dof_values_system.size() < fe.dofs_per_face) - dof_values_system.resize (fe.dofs_per_face, - Vector(fe.n_components())); - else - dof_values_system.resize (fe.dofs_per_face); - - function_map.find(boundary_component)->second - ->vector_value_list (dof_locations, dof_values_system); - - // enter those dofs - // into the list that - // match the - // component - // signature. avoid - // the usual - // complication that - // we can't just use - // *_system_to_component_index - // for non-primitive - // FEs - for (unsigned int i=0; iget_dof_indices (face_dofs, cell->active_fe_index()); + const std::vector > &dof_locations + = fe_values.get_quadrature_points (); - if (component_mask[component] == true) - boundary_values[face_dofs[i]] = dof_values_system[i](component); - } - } - else - // fe has only one component, - // so save some computations - { - // get only the one component that - // this function has - dof_values_scalar.resize (fe.dofs_per_face); - function_map.find(boundary_component)->second - ->value_list (dof_locations, dof_values_scalar, 0); + if (fe_is_system) + { + // resize + // array. avoid + // construction of a + // memory allocating + // temporary if + // possible + if (dof_values_system.size() < fe.dofs_per_face) + dof_values_system.resize (fe.dofs_per_face, + Vector(fe.n_components())); + else + dof_values_system.resize (fe.dofs_per_face); + + function_map.find(boundary_component)->second + ->vector_value_list (dof_locations, dof_values_system); + + // enter those dofs + // into the list that + // match the + // component + // signature. avoid + // the usual + // complication that + // we can't just use + // *_system_to_component_index + // for non-primitive + // FEs + for (unsigned int i=0; isecond + ->value_list (dof_locations, dof_values_scalar, 0); - // enter into list + // enter into list - for (unsigned int i=0; i boundary_values; interpolate_boundary_values (mapping, dof, function_map, boundary_values, component_mask_); - std::map::const_iterator boundary_value = + std::map::const_iterator boundary_value = boundary_values.begin(); for ( ; boundary_value !=boundary_values.end(); ++boundary_value) { - if (!constraints.is_constrained(boundary_value->first)) + if (constraints.can_store_line (boundary_value->first) + && + !constraints.is_constrained(boundary_value->first)) { constraints.add_line (boundary_value->first); constraints.set_inhomogeneity (boundary_value->first, @@ -2038,7 +2042,7 @@ VectorTools::project_boundary_values (const Mapping &mappin std::map boundary_values; project_boundary_values (mapping, dof, boundary_functions, q, boundary_values, component_mapping); - std::map::const_iterator boundary_value = + std::map::const_iterator boundary_value = boundary_values.begin(); for ( ; boundary_value !=boundary_values.end(); ++boundary_value) { @@ -2140,7 +2144,7 @@ namespace internal const Tensor<1,dim> &constraining_vector, ConstraintMatrix &constraints) { - + // choose the DoF that has the // largest component in the // constraining_vector as the @@ -2164,7 +2168,9 @@ namespace internal { if (std::fabs(constraining_vector[0]) > std::fabs(constraining_vector[1])) { - if (!constraints.is_constrained(dof_indices.dof_indices[0])) + if (!constraints.is_constrained(dof_indices.dof_indices[0]) + && + constraints.can_store_line(dof_indices.dof_indices[0])) { constraints.add_line (dof_indices.dof_indices[0]); @@ -2177,7 +2183,9 @@ namespace internal } else { - if (!constraints.is_constrained(dof_indices.dof_indices[1])) + if (!constraints.is_constrained(dof_indices.dof_indices[1]) + && + constraints.can_store_line(dof_indices.dof_indices[1])) { constraints.add_line (dof_indices.dof_indices[1]); @@ -2197,7 +2205,9 @@ namespace internal && (std::fabs(constraining_vector[0]) >= std::fabs(constraining_vector[2]))) { - if (!constraints.is_constrained(dof_indices.dof_indices[0])) + if (!constraints.is_constrained(dof_indices.dof_indices[0]) + && + constraints.can_store_line(dof_indices.dof_indices[0])) { constraints.add_line (dof_indices.dof_indices[0]); @@ -2219,7 +2229,9 @@ namespace internal && (std::fabs(constraining_vector[1]) >= std::fabs(constraining_vector[2]))) { - if (!constraints.is_constrained(dof_indices.dof_indices[1])) + if (!constraints.is_constrained(dof_indices.dof_indices[1]) + && + constraints.can_store_line(dof_indices.dof_indices[1])) { constraints.add_line (dof_indices.dof_indices[1]); @@ -2238,7 +2250,9 @@ namespace internal } else { - if (!constraints.is_constrained(dof_indices.dof_indices[2])) + if (!constraints.is_constrained(dof_indices.dof_indices[2]) + && + constraints.can_store_line(dof_indices.dof_indices[2])) { constraints.add_line (dof_indices.dof_indices[2]); @@ -2373,7 +2387,7 @@ namespace internal namespace internals { namespace VectorTools { - + // This function computes the // projection of the boundary // function on edges for 3D. @@ -2388,12 +2402,12 @@ namespace internals { std::vector& dof_values) { const unsigned int dim = 3; - + hp_fe_values.reinit (cell, (cell->active_fe_index () * GeometryInfo::faces_per_cell + face) * GeometryInfo::lines_per_face + line); - + // Initialize the required // objects. const FEValues& @@ -2402,20 +2416,20 @@ namespace internals { jacobians = fe_values.get_jacobians (); const std::vector >& quadrature_points = fe_values.get_quadrature_points (); - + std::vector > tangentials (fe_values.n_quadrature_points); std::vector > values (fe_values.n_quadrature_points, Vector (dim)); - + // Get boundary function values // at quadrature points. boundary_function.vector_value_list (quadrature_points, values); - + const std::vector >& reference_quadrature_points = fe_values.get_quadrature ().get_points (); const unsigned int superdegree = cell->get_fe ().degree; const unsigned int degree = superdegree - 1; - + // coordinate directions of // the edges of the face. const unsigned int @@ -2428,13 +2442,13 @@ namespace internals { { 0, 0, 2, 2 }, { 1, 1, 0, 0 }, { 1, 1, 0, 0 } }; - + // The interpolation for the // lowest order edge shape // functions is just the mean - // value of the tangential + // value of the tangential // components of the boundary - // function on the edge. + // function on the edge. for (unsigned int q_point = 0; q_point < fe_values.n_quadrature_points; ++q_point) { @@ -2443,7 +2457,7 @@ namespace internals { // the quadrature point. Point shifted_reference_point_1 = reference_quadrature_points[q_point]; Point shifted_reference_point_2 = reference_quadrature_points[q_point]; - + shifted_reference_point_1 (edge_coordinate_direction[face][line]) += 1e-13; shifted_reference_point_2 (edge_coordinate_direction[face][line]) -= 1e-13; tangentials[q_point] @@ -2457,7 +2471,7 @@ namespace internals { shifted_reference_point_2))); tangentials[q_point] /= std::sqrt (tangentials[q_point].square ()); - + // Compute the mean value. dof_values[line * superdegree] += (fe_values.JxW (q_point) @@ -2471,7 +2485,7 @@ namespace internals { + jacobians[q_point][2][edge_coordinate_direction[face][line]] * jacobians[q_point][2][edge_coordinate_direction[face][line]])); } - + // If there are also higher // order shape functions we // have still some work left. @@ -2480,7 +2494,7 @@ namespace internals { const FEValuesExtractors::Vector vec (first_vector_component); FullMatrix assembling_matrix (degree, fe_values.n_quadrature_points); Vector assembling_vector (fe_values.n_quadrature_points); - + // We set up a linear system // of equations to get the // values for the remaining @@ -2509,7 +2523,7 @@ namespace internals { jacobians[q_point][2][edge_coordinate_direction[face][line]] * jacobians[q_point][2][edge_coordinate_direction[face][line]])) * tangentials[q_point]; - + const Tensor<1, dim> shape_value = fe_values[vec].value (cell->get_fe () .face_to_cell_index (line * superdegree, face), @@ -2531,7 +2545,7 @@ namespace internals { (values[q_point] (2) - dof_values[line * superdegree] * shape_value[2]) * tmp[2]); - + for (unsigned int i = 0; i < degree; ++i) assembling_matrix (i, q_point) = fe_values[vec].value (cell->get_fe () @@ -2539,37 +2553,37 @@ namespace internals { face), q_point) * tmp; } - + FullMatrix cell_matrix (degree, degree); - + // Create the system matrix // by multiplying the // assembling matrix with its // transposed. assembling_matrix.mTmult (cell_matrix, assembling_matrix); - + FullMatrix cell_matrix_inv (degree, degree); // Compute its inverse. cell_matrix_inv.invert (cell_matrix); - + Vector cell_rhs (degree); - + // Create the system right // hand side vector by // multiplying the assembling // matrix with the assembling // vector. assembling_matrix.vmult (cell_rhs, assembling_vector); - + Vector solution (degree); - + cell_matrix_inv.vmult (solution, cell_rhs); // Store the computed values. for (unsigned int i = 0; i < degree; ++i) - dof_values[i + line * superdegree + 1] = solution (i); + dof_values[i + line * superdegree + 1] = solution (i); } } - + // dummy implementation of above // function for all other // dimensions @@ -2585,7 +2599,7 @@ namespace internals { { Assert (false, ExcInternalError ()); } - + // This function computes the // projection of the boundary // function on the interior of @@ -2607,10 +2621,10 @@ namespace internals { fe_values = hp_fe_values.get_present_fe_values (); const std::vector >& jacobians = fe_values.get_jacobians (); - + std::vector > values (fe_values.n_quadrature_points, Vector (dim)); - + switch (dim) { case 2: @@ -2619,29 +2633,29 @@ namespace internals { quadrature_points = fe_values.get_quadrature_points (); std::vector > tangentials (fe_values.n_quadrature_points); - + // Get boundary function // values at quadrature // points. boundary_function.vector_value_list (quadrature_points, values); - + const std::vector >& reference_quadrature_points = fe_values.get_quadrature ().get_points (); const unsigned int degree = cell->get_fe ().degree - 1; - + // coordinate directions // of the face. const unsigned int face_coordinate_direction[GeometryInfo::faces_per_cell] = { 1, 1, 0, 0 }; - + // The interpolation for // the lowest order face // shape functions is just // the mean value of the // tangential components // of the boundary function - // on the edge. + // on the edge. for (unsigned int q_point = 0; q_point < fe_values.n_quadrature_points; ++q_point) { @@ -2653,7 +2667,7 @@ namespace internals { = reference_quadrature_points[q_point]; Point shifted_reference_point_2 = reference_quadrature_points[q_point]; - + shifted_reference_point_1 (face_coordinate_direction[face]) += 1e-13; shifted_reference_point_2 (face_coordinate_direction[face]) @@ -2681,7 +2695,7 @@ namespace internals { + jacobians[q_point][1][face_coordinate_direction[face]] * jacobians[q_point][1][face_coordinate_direction[face]]); } - + // If there are also // higher order shape // functions we have @@ -2692,7 +2706,7 @@ namespace internals { FullMatrix assembling_matrix (degree, fe_values.n_quadrature_points); Vector assembling_vector (fe_values.n_quadrature_points); - + // We set up a // linear system // of equations to @@ -2723,12 +2737,12 @@ namespace internals { + jacobians[q_point][1][face_coordinate_direction[face]] * jacobians[q_point][1][face_coordinate_direction[face]])) * tangentials[q_point]; - + const Tensor<1, dim> shape_value = fe_values[vec].value (cell->get_fe () .face_to_cell_index (0, face), q_point); - + assembling_vector (q_point) = (values[q_point] (0) - dof_values[0] * shape_value[0]) * tmp[0] @@ -2736,7 +2750,7 @@ namespace internals { (values[q_point] (1) - dof_values[1] * shape_value[1]) * tmp[1]; - + // In the weak // form the // right hand @@ -2751,22 +2765,22 @@ namespace internals { .face_to_cell_index (i + 1, face), q_point) * tmp; } - + FullMatrix cell_matrix (degree, degree); - + // Create the system // matrix by multiplying // the assembling // matrix with its // transposed. assembling_matrix.mTmult (cell_matrix, assembling_matrix); - + FullMatrix cell_matrix_inv (degree, degree); // Compute its inverse. cell_matrix_inv.invert (cell_matrix); - + Vector cell_rhs (degree); - + // Create the system // right hand side // vector by @@ -2775,30 +2789,30 @@ namespace internals { // with the assembling // vector. assembling_matrix.vmult (cell_rhs, assembling_vector); - + Vector solution (degree); - + cell_matrix_inv.vmult (solution, cell_rhs); - + // Store the computed // values. for (unsigned int i = 0; i < degree; ++i) dof_values[i + 1] = solution (i); } - + break; } - + case 3: { const std::vector >& quadrature_points = fe_values.get_quadrature_points (); - + // Get boundary function // values at quadrature // points. boundary_function.vector_value_list (quadrature_points, values); - + const FEValuesExtractors::Vector vec (first_vector_component); const unsigned int superdegree = cell->get_fe ().degree; const unsigned int degree = superdegree - 1; @@ -2812,7 +2826,7 @@ namespace internals { FullMatrix cell_matrix_inv (assembling_matrix.m (), assembling_matrix.m ()); Vector solution (cell_matrix.m ()); - + // Get coordinate directions // of the face. const unsigned int @@ -2831,7 +2845,7 @@ namespace internals { { 0, 1 }, { 1, 0 }, { 1, 0 } }; - + // The projection is // divided into two steps. // In the first step we @@ -2864,17 +2878,17 @@ namespace internals { // interpolated part // on the edges. Tensor<1, dim> tmp; - + for (unsigned int d = 0; d < dim; ++d) tmp[d] = values[q_point] (d); - + for (unsigned int i = 0; i < 2; ++i) for (unsigned int j = 0; j <= degree; ++j) tmp -= dof_values[(i + 2 * local_face_coordinate_directions[face][0]) * superdegree + j] * fe_values[vec].value (cell->get_fe ().face_to_cell_index ((i + 2 * local_face_coordinate_directions[face][0]) * superdegree + j, face), q_point); - + const double JxW = std::sqrt (fe_values.JxW (q_point) / ((jacobians[q_point][0][global_face_coordinate_directions[face][0]] @@ -2894,7 +2908,7 @@ namespace internals { + jacobians[q_point][2][global_face_coordinate_directions[face][1]] * jacobians[q_point][2][global_face_coordinate_directions[face][1]]))); - + // In the weak form // the right hand // side function @@ -2906,7 +2920,7 @@ namespace internals { // the face. for (unsigned int d = 0; d < dim; ++d) assembling_vector (dim * q_point + d) = JxW * tmp[d]; - + for (unsigned int i = 0; i <= degree; ++i) for (unsigned int j = 0; j < degree; ++j) { @@ -2920,14 +2934,14 @@ namespace internals { + GeometryInfo::lines_per_face, face), q_point)); - + for (unsigned int d = 0; d < dim; ++d) assembling_matrix (i * degree + j, dim * q_point + d) = shape_value[d]; } } - + // Create the system // matrix by // multiplying the @@ -2945,7 +2959,7 @@ namespace internals { cell_matrix_inv.invert (cell_matrix); assembling_matrix.vmult (cell_rhs, assembling_vector); cell_matrix_inv.vmult (solution, cell_rhs); - + // Store the computed // values. for (unsigned int i = 0; i <= degree; ++i) @@ -2953,7 +2967,7 @@ namespace internals { dof_values[(i + GeometryInfo::lines_per_face) * degree + j + GeometryInfo::lines_per_face] = solution (i * degree + j); - + // Now we do the // same as above // with the vertical @@ -2964,10 +2978,10 @@ namespace internals { q_point < fe_values.n_quadrature_points; ++q_point) { Tensor<1, dim> tmp; - + for (unsigned int d = 0; d < dim; ++d) tmp[d] = values[q_point] (d); - + for (unsigned int i = 0; i < 2; ++i) for (unsigned int j = 0; j <= degree; ++j) tmp @@ -2975,7 +2989,7 @@ namespace internals { * fe_values[vec].value (cell->get_fe ().face_to_cell_index ((i + 2 * local_face_coordinate_directions[face][1]) * superdegree + j, face), q_point); - + const double JxW = std::sqrt (fe_values.JxW (q_point) / ((jacobians[q_point][0][global_face_coordinate_directions[face][0]] @@ -2995,10 +3009,10 @@ namespace internals { + jacobians[q_point][2][global_face_coordinate_directions[face][1]] * jacobians[q_point][2][global_face_coordinate_directions[face][1]]))); - + for (unsigned int d = 0; d < dim; ++d) assembling_vector (dim * q_point + d) = JxW * tmp[d]; - + for (unsigned int i = 0; i < degree; ++i) for (unsigned int j = 0; j <= degree; ++j) { @@ -3007,34 +3021,34 @@ namespace internals { * fe_values[vec].value (cell->get_fe ().face_to_cell_index ((i + degree + GeometryInfo::lines_per_face) * superdegree + j, face), q_point)); - + for (unsigned int d = 0; d < dim; ++d) assembling_matrix (i * superdegree + j, dim * q_point + d) = shape_value[d]; } } - + assembling_matrix.mTmult (cell_matrix, assembling_matrix); cell_matrix_inv.invert (cell_matrix); assembling_matrix.vmult (cell_rhs, assembling_vector); cell_matrix_inv.vmult (solution, cell_rhs); - + for (unsigned int i = 0; i < degree; ++i) for (unsigned int j = 0; j <= degree; ++j) dof_values[(i + degree + GeometryInfo::lines_per_face) * superdegree + j] = solution (i * superdegree + j); - + break; } - + default: Assert (false, ExcNotImplemented ()); } } } } - - + + template @@ -3093,22 +3107,22 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, hp::FECollection fe_collection (dof_handler.get_fe ()); hp::MappingCollection mapping_collection (mapping); hp::QCollection face_quadrature_collection; - + for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) face_quadrature_collection.push_back (QProjector::project_to_face (reference_face_quadrature, face)); - + hp::FEValues fe_face_values (mapping_collection, fe_collection, face_quadrature_collection, update_jacobians | update_JxW_values | update_quadrature_points | update_values); - + std::vector dof_values (dofs_per_face); std::vector face_dof_indices (dofs_per_face); typename DoFHandler::active_cell_iterator cell = dof_handler.begin_active (); - + switch (dim) { case 2: @@ -3125,10 +3139,10 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, typedef FiniteElement FEL; AssertThrow (dynamic_cast*> (&cell->get_fe ()) != 0, typename FEL::ExcInterpolationNotImplemented ()); - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) dof_values[dof] = 0.0; - + // Compute the // projection of the // boundary function on @@ -3136,10 +3150,10 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, internals::VectorTools ::compute_face_projection (cell, face, fe_face_values, boundary_function, - first_vector_component, dof_values); + first_vector_component, dof_values); cell->face (face)->get_dof_indices (face_dof_indices, cell->active_fe_index ()); - + // Add the computed // constraints to the // constraint matrix, @@ -3150,29 +3164,29 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, if (!(constraints.is_constrained (face_dof_indices[dof]))) { constraints.add_line (face_dof_indices[dof]); - + if (std::abs (dof_values[dof]) > 1e-14) constraints.set_inhomogeneity (face_dof_indices[dof], dof_values[dof]); } } - + break; } - + case 3: { const QGauss reference_edge_quadrature (2 * superdegree); const unsigned int degree = superdegree - 1; const unsigned int n_dofs = dof_handler.n_dofs (); hp::QCollection edge_quadrature_collection; - + for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) for (unsigned int line = 0; line < GeometryInfo::lines_per_face; ++line) edge_quadrature_collection.push_back (QProjector::project_to_face (QProjector::project_to_face (reference_edge_quadrature, line), face)); - + hp::FEValues fe_edge_values (mapping_collection, fe_collection, edge_quadrature_collection, update_jacobians | @@ -3181,10 +3195,10 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, update_values); std::vector computed_constraints (n_dofs); std::vector projected_dofs (n_dofs); - + for (unsigned int dof = 0; dof < n_dofs; ++dof) projected_dofs[dof] = -1; - + for (; cell != dof_handler.end (); ++cell) if (cell->at_boundary ()) for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) @@ -3197,13 +3211,13 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, typedef FiniteElement FEL; AssertThrow (dynamic_cast*> (&cell->get_fe ()) != 0, typename FEL::ExcInterpolationNotImplemented ()); - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) dof_values[dof] = 0.0; - + cell->face (face)->get_dof_indices (face_dof_indices, cell->active_fe_index ()); - + // First we compute the // projection on the // edges. @@ -3242,7 +3256,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, dof < (line + 1) * superdegree; ++dof) projected_dofs[face_dof_indices[dof]] = degree; } - + // If we have // computed the // values in a @@ -3255,7 +3269,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, dof < (line + 1) * superdegree; ++dof) dof_values[dof] = computed_constraints[face_dof_indices[dof]]; } - + // If there are higher // order shape // functions, there is @@ -3273,7 +3287,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, boundary_function, first_vector_component, dof_values); - + // Mark the projected // degrees of // freedom. @@ -3281,7 +3295,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, dof < dofs_per_face; ++dof) projected_dofs[face_dof_indices[dof]] = degree; } - + // Store the computed // values in the global // vector. @@ -3289,7 +3303,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, if (std::abs (dof_values[dof]) > 1e-14) computed_constraints[face_dof_indices[dof]] = dof_values[dof]; } - + // Add the computed constraints // to the constraint matrix, if // the degree of freedom is not @@ -3301,7 +3315,7 @@ project_boundary_values_curl_conforming (const DoFHandler& dof_handler, constraints.set_inhomogeneity (dof, computed_constraints[dof]); } } - + default: Assert (false, ExcNotImplemented ()); } @@ -3321,17 +3335,17 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, { hp::FECollection fe_collection (dof_handler.get_fe ()); hp::QCollection face_quadrature_collection; - + for (unsigned int i = 0; i < fe_collection.size (); ++i) { const QGauss reference_face_quadrature (2 * fe_collection[i].degree); - + for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) face_quadrature_collection.push_back (QProjector::project_to_face (reference_face_quadrature, face)); } - + hp::FEValues fe_face_values (mapping_collection, fe_collection, face_quadrature_collection, update_jacobians | @@ -3341,7 +3355,7 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, std::vector dof_values; std::vector face_dof_indices; typename hp::DoFHandler::active_cell_iterator cell = dof_handler.begin_active (); - + switch (dim) { case 2: @@ -3358,14 +3372,14 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, typedef FiniteElement FEL; AssertThrow (dynamic_cast *> (&cell->get_fe ()) != 0, typename FEL::ExcInterpolationNotImplemented ()); - + const unsigned int dofs_per_face = cell->get_fe ().dofs_per_face; - + dof_values.resize (dofs_per_face); - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) dof_values[dof] = 0.0; - + internals::VectorTools ::compute_face_projection (cell, face, fe_face_values, boundary_function, @@ -3374,30 +3388,30 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, face_dof_indices.resize (dofs_per_face); cell->face (face)->get_dof_indices (face_dof_indices, cell->active_fe_index ()); - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) if (!(constraints.is_constrained (face_dof_indices[dof]))) { constraints.add_line (face_dof_indices[dof]); - + if (std::abs (dof_values[dof]) > 1e-14) constraints.set_inhomogeneity (face_dof_indices[dof], dof_values[dof]); } } - + break; } - + case 3: { const unsigned int n_dofs = dof_handler.n_dofs (); hp::QCollection edge_quadrature_collection; - + for (unsigned int i = 0; i < fe_collection.size (); ++i) { const QGauss reference_edge_quadrature (2 * fe_collection[i].degree); - + for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) for (unsigned int line = 0; line < GeometryInfo::lines_per_face; ++line) edge_quadrature_collection.push_back @@ -3405,7 +3419,7 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, (QProjector::project_to_face (reference_edge_quadrature, line), face)); } - + hp::FEValues fe_edge_values (mapping_collection, fe_collection, edge_quadrature_collection, update_jacobians | @@ -3414,10 +3428,10 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, update_values); std::vector computed_constraints (n_dofs); std::vector projected_dofs (n_dofs); - + for (unsigned int dof = 0; dof < n_dofs; ++dof) projected_dofs[dof] = -1; - + for (; cell != dof_handler.end (); ++cell) if (cell->at_boundary ()) for (unsigned int face = 0; face < GeometryInfo::faces_per_cell; ++face) @@ -3430,20 +3444,20 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, typedef FiniteElement FEL; AssertThrow (dynamic_cast *> (&cell->get_fe ()) != 0, typename FEL::ExcInterpolationNotImplemented ()); - + const unsigned int superdegree = cell->get_fe ().degree; const unsigned int degree = superdegree - 1; const unsigned int dofs_per_face = cell->get_fe ().dofs_per_face; - + dof_values.resize (dofs_per_face); - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) dof_values[dof] = 0.0; - + face_dof_indices.resize (dofs_per_face); cell->face (face)->get_dof_indices (face_dof_indices, cell->active_fe_index ()); - + for (unsigned int line = 0; line < GeometryInfo::lines_per_face; ++line) { @@ -3457,18 +3471,18 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, boundary_function, first_vector_component, dof_values); - + for (unsigned int dof = line * superdegree; dof < (line + 1) * superdegree; ++dof) projected_dofs[face_dof_indices[dof]] = degree; } - + else for (unsigned int dof = line * superdegree; dof < (line + 1) * superdegree; ++dof) dof_values[dof] = computed_constraints[face_dof_indices[dof]]; } - + if (degree > 0) { internals::VectorTools @@ -3476,17 +3490,17 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, boundary_function, first_vector_component, dof_values); - + for (unsigned int dof = GeometryInfo::lines_per_face * superdegree; dof < dofs_per_face; ++dof) projected_dofs[face_dof_indices[dof]] = degree; } - + for (unsigned int dof = 0; dof < dofs_per_face; ++dof) if (std::abs (dof_values[dof]) > 1e-14) computed_constraints[face_dof_indices[dof]] = dof_values[dof]; } - + for (unsigned int dof = 0; dof < n_dofs; ++dof) if ((projected_dofs[dof] != -1) && !(constraints.is_constrained (dof))) { @@ -3494,7 +3508,7 @@ project_boundary_values_curl_conforming (const hp::DoFHandler& dof_handler, constraints.set_inhomogeneity (dof, computed_constraints[dof]); } } - + default: Assert (false, ExcNotImplemented ()); } @@ -3558,67 +3572,68 @@ compute_no_normal_flux_constraints (const DH &dof_handler, cell = dof_handler.begin_active(), endc = dof_handler.end(); for (; cell!=endc; ++cell) - for (unsigned int face_no=0; face_no < GeometryInfo::faces_per_cell; - ++face_no) - if (boundary_ids.find(cell->face(face_no)->boundary_indicator()) - != boundary_ids.end()) - { - typename DH::face_iterator face = cell->face(face_no); - - // get the indices of the - // dofs on this cell... - face->get_dof_indices (face_dofs, cell->active_fe_index()); - - // ...and the normal - // vectors at the locations - // where they are defined: - const std::vector > & - unit_support_points = fe.get_unit_face_support_points(); - Quadrature aux_quad (unit_support_points); - FEFaceValues fe_values (mapping, fe, aux_quad, - update_normal_vectors); - fe_values.reinit(cell, face_no); - - // then identify which of - // them correspond to the - // selected set of vector - // components - for (unsigned int i=0; i vector_dofs; - vector_dofs.dof_indices[0] = face_dofs[i]; + if (!cell->is_artificial()) + for (unsigned int face_no=0; face_no < GeometryInfo::faces_per_cell; + ++face_no) + if (boundary_ids.find(cell->face(face_no)->boundary_indicator()) + != boundary_ids.end()) + { + typename DH::face_iterator face = cell->face(face_no); - for (unsigned int k=0; k= - first_vector_component) - && - (fe.face_system_to_component_index(k).first < - first_vector_component + dim)) - vector_dofs.dof_indices[fe.face_system_to_component_index(k).first - - first_vector_component] - = face_dofs[k]; - - for (unsigned int d=0; dget_dof_indices (face_dofs, cell->active_fe_index()); + + // ...and the normal + // vectors at the locations + // where they are defined: + const std::vector > & + unit_support_points = fe.get_unit_face_support_points(); + Quadrature aux_quad (unit_support_points); + FEFaceValues fe_values (mapping, fe, aux_quad, + update_normal_vectors); + fe_values.reinit(cell, face_no); + + // then identify which of + // them correspond to the + // selected set of vector + // components + for (unsigned int i=0; i vector_dofs; + vector_dofs.dof_indices[0] = face_dofs[i]; + + for (unsigned int k=0; k= + first_vector_component) + && + (fe.face_system_to_component_index(k).first < + first_vector_component + dim)) + vector_dofs.dof_indices[fe.face_system_to_component_index(k).first - + first_vector_component] + = face_dofs[k]; + +/* for (unsigned int d=0; d &dof_handler, // constrained for (unsigned int i=0; ifirst.dof_indices[i])) + ->first.dof_indices[i]) + && + constraints.can_store_line( + same_dof_range[0]->first.dof_indices[i])) { constraints.add_line (same_dof_range[0]->first.dof_indices[i]); // no add_entries here diff --git a/deal.II/deal.II/source/distributed/grid_refinement.cc b/deal.II/deal.II/source/distributed/grid_refinement.cc new file mode 100644 index 0000000000..fbb0e3cc4b --- /dev/null +++ b/deal.II/deal.II/source/distributed/grid_refinement.cc @@ -0,0 +1,775 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + + + +#include +#include +#include +#include +#include +#include +#include + +#ifdef DEAL_II_USE_P4EST + +#include +#include +#include +#include + +#include + +#include +#include + + +DEAL_II_NAMESPACE_OPEN + + +namespace +{ + template + inline + number + max_element (const Vector &criteria) + { + return *std::max_element(criteria.begin(), criteria.end()); + } + + + + template + inline + number + min_element (const Vector &criteria) + { + return *std::min_element(criteria.begin(), criteria.end()); + } + + + /** + * Compute the global max and min + * of the criteria vector. These + * are returned only on the + * processor with rank zero, all + * others get a pair of zeros. + */ + template + std::pair + compute_global_min_and_max_at_root (const Vector &criteria, + MPI_Comm mpi_communicator) + { + // we'd like to compute the + // global max and min from the + // local ones in one MPI + // communication. we can do that + // by taking the elementwise + // minimum of the local min and + // the negative maximum over all + // processors + + const double local_min = min_element (criteria), + local_max = max_element (criteria); + double comp[2] = { local_min, -local_max }; + double result[2] = { 0, 0 }; + + // compute the minimum on + // processor zero + MPI_Reduce (&comp, &result, 2, MPI_DOUBLE, + MPI_MIN, 0, mpi_communicator); + + // make sure only processor zero + // got something + if (Utilities::System::get_this_mpi_process (mpi_communicator) != 0) + Assert ((result[0] == 0) && (result[1] == 0), + ExcInternalError()); + + return std::make_pair (result[0], -result[1]); + } + + + + /** + * Compute the global sum over the elements + * of the vectors passed to this function + * on all processors. This number is + * returned only on the processor with rank + * zero, all others get zero. + */ + template + double + compute_global_sum (const Vector &criteria, + MPI_Comm mpi_communicator) + { + double my_sum = std::accumulate (criteria.begin(), + criteria.end(), + /* do accumulation in the correct data type: */ + number()); + + double result = 0; + // compute the minimum on + // processor zero + MPI_Reduce (&my_sum, &result, 1, MPI_DOUBLE, + MPI_SUM, 0, mpi_communicator); + + // make sure only processor zero + // got something + if (Utilities::System::get_this_mpi_process (mpi_communicator) != 0) + Assert (result == 0, ExcInternalError()); + + return result; + } + + + + /** + * Given a vector of refinement criteria + * for all cells of a mesh (locally owned + * or not), extract those that pertain to + * locally owned cells. + */ + template + void + get_locally_owned_indicators (const parallel::distributed::Triangulation &tria, + const Vector &criteria, + dealii::Vector &locally_owned_indicators) + { + Assert (locally_owned_indicators.size() == tria.n_locally_owned_active_cells(), + ExcInternalError()); + + unsigned int active_index = 0; + unsigned int owned_index = 0; + for (typename Triangulation::active_cell_iterator + cell = tria.begin_active(); + cell != tria.end(); ++cell, ++active_index) + if (cell->subdomain_id() == tria.locally_owned_subdomain()) + { + locally_owned_indicators(owned_index) + = criteria(active_index); + ++owned_index; + } + Assert (owned_index == tria.n_locally_owned_active_cells(), + ExcInternalError()); + Assert ((active_index == tria.dealii::Triangulation::n_active_cells()), + ExcInternalError()); + } + + + // we compute refinement + // thresholds by bisection of the + // interval spanned by the + // smallest and largest error + // indicator. this leads to a + // small problem: if, for + // example, we want to coarsen + // zero per cent of the cells, + // then we need to pick a + // threshold equal to the + // smallest indicator, but of + // course the bisection algorithm + // can never find a threshold + // equal to one of the end points + // of the interval. So we + // slightly increase the interval + // before we even start + void adjust_interesting_range (double (&interesting_range)[2]) + { + Assert (interesting_range[0] <= interesting_range[1], + ExcInternalError()); + + if (interesting_range[0] > 0) + interesting_range[0] *= 0.99; + else + interesting_range[0] + -= 0.01 * (interesting_range[1] - interesting_range[0]); + + if (interesting_range[1] > 0) + interesting_range[1] *= 1.01; + else + interesting_range[1] + += 0.01 * (interesting_range[1] - interesting_range[0]); + } + + + + /** + * Given a vector of criteria and bottom + * and top thresholds for coarsening and + * refinement, mark all those cells that we + * locally own as appropriate for + * coarsening or refinement. + */ + template + void + mark_cells (parallel::distributed::Triangulation &tria, + const Vector &criteria, + const double top_threshold, + const double bottom_threshold) + { + dealii::GridRefinement::refine (tria, criteria, top_threshold); + dealii::GridRefinement::coarsen (tria, criteria, bottom_threshold); + + // as a final good measure, + // delete all flags again + // from cells that we don't + // locally own + for (typename Triangulation::active_cell_iterator + cell = tria.begin_active(); + cell != tria.end(); ++cell) + if (cell->subdomain_id() != tria.locally_owned_subdomain()) + { + cell->clear_refine_flag (); + cell->clear_coarsen_flag (); + } + } + + + + + namespace RefineAndCoarsenFixedNumber + { + /** + * Compute a threshold value so + * that exactly n_target_cells have + * a value that is larger. + */ + template + number + master_compute_threshold (const Vector &criteria, + const std::pair global_min_and_max, + const unsigned int n_target_cells, + MPI_Comm mpi_communicator) + { + double interesting_range[2] = { global_min_and_max.first, + global_min_and_max.second }; + adjust_interesting_range (interesting_range); + + unsigned int iteration = 0; + + do + { + MPI_Bcast (&interesting_range[0], 2, MPI_DOUBLE, + 0, mpi_communicator); + + if (interesting_range[0] == interesting_range[1]) + return interesting_range[0]; + + const double test_threshold + = (interesting_range[0] > 0 + ? + std::sqrt(interesting_range[0] * + interesting_range[1]) + : + (interesting_range[0] + interesting_range[1]) / 2); + + // count how many of our own + // elements would be above + // this threshold and then + // add to it the number for + // all the others + unsigned int + my_count = std::count_if (criteria.begin(), + criteria.end(), + std::bind2nd (std::greater(), + test_threshold)); + + unsigned int total_count; + MPI_Reduce (&my_count, &total_count, 1, MPI_UNSIGNED, + MPI_SUM, 0, mpi_communicator); + + // now adjust the range. if + // we have to many cells, we + // take the upper half of the + // previous range, otherwise + // the lower half. if we have + // hit the right number, then + // set the range to the exact + // value + if (total_count > n_target_cells) + interesting_range[0] = test_threshold; + else if (total_count < n_target_cells) + interesting_range[1] = test_threshold; + else + interesting_range[0] = interesting_range[1] = test_threshold; + + // terminate the iteration + // after 10 go-arounds. this + // is necessary because + // oftentimes error + // indicators on cells have + // exactly the same value, + // and so there may not be a + // particular value that cuts + // the indicators in such a + // way that we can achieve + // the desired number of + // cells. using a max of 10 + // iterations means that we + // terminate the iteration + // after 10 steps if the + // indicators were perfectly + // badly distributed, and we + // make at most a mistake of + // 1/2^10 in the number of + // cells flagged if + // indicators are perfectly + // equidistributed + ++iteration; + if (iteration == 25) + interesting_range[0] = interesting_range[1] = test_threshold; + } + while (true); + + Assert (false, ExcInternalError()); + return -1; + } + + + /** + * The corresponding function to + * the one above, to be run on the + * slaves. + */ + template + number + slave_compute_threshold (const Vector &criteria, + MPI_Comm mpi_communicator) + { + do + { + double interesting_range[2] = { -1, -1 }; + MPI_Bcast (&interesting_range[0], 2, MPI_DOUBLE, + 0, mpi_communicator); + + if (interesting_range[0] == interesting_range[1]) + return interesting_range[0]; + + // count how many elements + // there are that are bigger + // than the following trial + // threshold + const double test_threshold + = (interesting_range[0] > 0 + ? + std::exp((std::log(interesting_range[0]) + + std::log(interesting_range[1])) / 2) + : + (interesting_range[0] + interesting_range[1]) / 2); + unsigned int + my_count = std::count_if (criteria.begin(), + criteria.end(), + std::bind2nd (std::greater(), + test_threshold)); + + MPI_Reduce (&my_count, 0, 1, MPI_UNSIGNED, + MPI_SUM, 0, mpi_communicator); + } + while (true); + + Assert (false, ExcInternalError()); + return -1; + } + } + + + + namespace RefineAndCoarsenFixedFraction + { + /** + * Compute a threshold value so + * that exactly n_target_cells have + * a value that is larger. + */ + template + number + master_compute_threshold (const Vector &criteria, + const std::pair global_min_and_max, + const double target_error, + MPI_Comm mpi_communicator) + { + double interesting_range[2] = { global_min_and_max.first, + global_min_and_max.second }; + adjust_interesting_range (interesting_range); + + unsigned int iteration = 0; + + do + { + MPI_Bcast (&interesting_range[0], 2, MPI_DOUBLE, + 0, mpi_communicator); + + if (interesting_range[0] == interesting_range[1]) + return interesting_range[0]; + + const double test_threshold + = (interesting_range[0] > 0 + ? + std::exp((std::log(interesting_range[0]) + + std::log(interesting_range[1])) / 2) + : + (interesting_range[0] + interesting_range[1]) / 2); + + // accumulate the error of those + // our own elements above this + // threshold and then add to it the + // number for all the others + double my_error = 0; + for (unsigned int i=0; i test_threshold) + my_error += criteria(i); + + double total_error; + MPI_Reduce (&my_error, &total_error, 1, MPI_DOUBLE, + MPI_SUM, 0, mpi_communicator); + + // now adjust the range. if + // we have to many cells, we + // take the upper half of the + // previous range, otherwise + // the lower half. if we have + // hit the right number, then + // set the range to the exact + // value + if (total_error > target_error) + interesting_range[0] = test_threshold; + else if (total_error < target_error) + interesting_range[1] = test_threshold; + else + interesting_range[0] = interesting_range[1] = test_threshold; + + // terminate the iteration + // after 10 go-arounds. this + // is necessary because + // oftentimes error + // indicators on cells have + // exactly the same value, + // and so there may not be a + // particular value that cuts + // the indicators in such a + // way that we can achieve + // the desired number of + // cells. using a max of 10 + // iterations means that we + // terminate the iteration + // after 10 steps if the + // indicators were perfectly + // badly distributed, and we + // make at most a mistake of + // 1/2^10 in the number of + // cells flagged if + // indicators are perfectly + // equidistributed + ++iteration; + if (iteration == 25) + interesting_range[0] = interesting_range[1] = test_threshold; + } + while (true); + + Assert (false, ExcInternalError()); + return -1; + } + + + /** + * The corresponding function to + * the one above, to be run on the + * slaves. + */ + template + number + slave_compute_threshold (const Vector &criteria, + MPI_Comm mpi_communicator) + { + do + { + double interesting_range[2] = { -1, -1 }; + MPI_Bcast (&interesting_range[0], 2, MPI_DOUBLE, + 0, mpi_communicator); + + if (interesting_range[0] == interesting_range[1]) + return interesting_range[0]; + + // count how many elements + // there are that are bigger + // than the following trial + // threshold + const double test_threshold + = (interesting_range[0] > 0 + ? + std::exp((std::log(interesting_range[0]) + + std::log(interesting_range[1])) / 2) + : + (interesting_range[0] + interesting_range[1]) / 2); + + double my_error = 0; + for (unsigned int i=0; i test_threshold) + my_error += criteria(i); + + MPI_Reduce (&my_error, 0, 1, MPI_DOUBLE, + MPI_SUM, 0, mpi_communicator); + } + while (true); + + Assert (false, ExcInternalError()); + return -1; + } + } +} + + + +namespace parallel +{ + namespace distributed + { + namespace GridRefinement + { + template + void + refine_and_coarsen_fixed_number ( + parallel::distributed::Triangulation &tria, + const Vector &criteria, + const double top_fraction_of_cells, + const double bottom_fraction_of_cells) + { + Assert ((top_fraction_of_cells>=0) && (top_fraction_of_cells<=1), + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert ((bottom_fraction_of_cells>=0) && (bottom_fraction_of_cells<=1), + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert (top_fraction_of_cells+bottom_fraction_of_cells <= 1, + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert (criteria.is_non_negative (), + dealii::GridRefinement::ExcNegativeCriteria()); + + // first extract from the + // vector of indicators the + // ones that correspond to + // cells that we locally own + dealii::Vector + locally_owned_indicators (tria.n_locally_owned_active_cells()); + get_locally_owned_indicators (tria, + criteria, + locally_owned_indicators); + + MPI_Comm mpi_communicator = tria.get_communicator (); + + // figure out the global + // max and min of the + // indicators. we don't + // need it here, but it's a + // collective communication + // call + const std::pair global_min_and_max + = compute_global_min_and_max_at_root (locally_owned_indicators, + mpi_communicator); + + // from here on designate a + // master and slaves + double top_threshold, bottom_threshold; + if (Utilities::System::get_this_mpi_process (mpi_communicator) == 0) + { + // this is the master + // processor + top_threshold + = + RefineAndCoarsenFixedNumber:: + master_compute_threshold (locally_owned_indicators, + global_min_and_max, + static_cast + (top_fraction_of_cells * + tria.n_global_active_cells()), + mpi_communicator); + + // compute bottom + // threshold only if + // necessary. otherwise + // use a threshold lower + // than the smallest + // value we have locally + if (bottom_fraction_of_cells > 0) + bottom_threshold + = + RefineAndCoarsenFixedNumber:: + master_compute_threshold (locally_owned_indicators, + global_min_and_max, + static_cast + ((1-bottom_fraction_of_cells) * + tria.n_global_active_cells()), + mpi_communicator); + else + { + bottom_threshold = *std::min_element (criteria.begin(), + criteria.end()); + bottom_threshold -= std::fabs(bottom_threshold); + } + } + else + { + // this is a slave + // processor + top_threshold + = + RefineAndCoarsenFixedNumber:: + slave_compute_threshold (locally_owned_indicators, + mpi_communicator); + // compute bottom + // threshold only if + // necessary + if (bottom_fraction_of_cells > 0) + bottom_threshold + = + RefineAndCoarsenFixedNumber:: + slave_compute_threshold (locally_owned_indicators, + mpi_communicator); + else + { + bottom_threshold = *std::min_element (criteria.begin(), + criteria.end()); + bottom_threshold -= std::fabs(bottom_threshold); + } + } + + // now refine the mesh + mark_cells (tria, criteria, top_threshold, bottom_threshold); + } + + + template + void + refine_and_coarsen_fixed_fraction ( + parallel::distributed::Triangulation &tria, + const Vector &criteria, + const double top_fraction_of_error, + const double bottom_fraction_of_error) + { + Assert ((top_fraction_of_error>=0) && (top_fraction_of_error<=1), + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert ((bottom_fraction_of_error>=0) && (bottom_fraction_of_error<=1), + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert (top_fraction_of_error+bottom_fraction_of_error <= 1, + dealii::GridRefinement::ExcInvalidParameterValue()); + Assert (criteria.is_non_negative (), + dealii::GridRefinement::ExcNegativeCriteria()); + + // first extract from the + // vector of indicators the + // ones that correspond to + // cells that we locally own + dealii::Vector + locally_owned_indicators (tria.n_locally_owned_active_cells()); + get_locally_owned_indicators (tria, + criteria, + locally_owned_indicators); + + MPI_Comm mpi_communicator = tria.get_communicator (); + + // figure out the global + // max and min of the + // indicators. we don't + // need it here, but it's a + // collective communication + // call + const std::pair global_min_and_max + = compute_global_min_and_max_at_root (locally_owned_indicators, + mpi_communicator); + + const double total_error + = compute_global_sum (locally_owned_indicators, + mpi_communicator); + + // from here on designate a + // master and slaves + double top_threshold, bottom_threshold; + if (Utilities::System::get_this_mpi_process (mpi_communicator) == 0) + { + // this is the master + // processor + top_threshold + = + RefineAndCoarsenFixedFraction:: + master_compute_threshold (locally_owned_indicators, + global_min_and_max, + static_cast + (top_fraction_of_error * + total_error), + mpi_communicator); + + // compute bottom + // threshold only if + // necessary. otherwise + // use a threshold lower + // than the smallest + // value we have locally + if (bottom_fraction_of_error > 0) + bottom_threshold + = + RefineAndCoarsenFixedFraction:: + master_compute_threshold (locally_owned_indicators, + global_min_and_max, + static_cast + ((1-bottom_fraction_of_error) * + total_error), + mpi_communicator); + else + { + bottom_threshold = *std::min_element (criteria.begin(), + criteria.end()); + bottom_threshold -= std::fabs(bottom_threshold); + } + } + else + { + // this is a slave + // processor + top_threshold + = + RefineAndCoarsenFixedFraction:: + slave_compute_threshold (locally_owned_indicators, + mpi_communicator); + + // compute bottom + // threshold only if + // necessary. otherwise + // use a threshold lower + // than the smallest + // value we have locally + if (bottom_fraction_of_error > 0) + bottom_threshold + = + RefineAndCoarsenFixedFraction:: + slave_compute_threshold (locally_owned_indicators, + mpi_communicator); + else + { + bottom_threshold = *std::min_element (criteria.begin(), + criteria.end()); + bottom_threshold -= std::fabs(bottom_threshold); + } + } + + // now refine the mesh + mark_cells (tria, criteria, top_threshold, bottom_threshold); + } + } + } +} + + +// explicit instantiations +#include "grid_refinement.inst" + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/deal.II/deal.II/source/distributed/grid_refinement.inst.in b/deal.II/deal.II/source/distributed/grid_refinement.inst.in new file mode 100644 index 0000000000..a7454dbc75 --- /dev/null +++ b/deal.II/deal.II/source/distributed/grid_refinement.inst.in @@ -0,0 +1,44 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + + + +for (S : REAL_SCALARS) +{ +#if deal_II_dimension != 1 +namespace parallel +\{ + namespace distributed + \{ + namespace GridRefinement + \{ + template + void + refine_and_coarsen_fixed_number,deal_II_dimension> + (parallel::distributed::Triangulation &, + const dealii::Vector &, + const double, + const double); + + template + void + refine_and_coarsen_fixed_fraction,deal_II_dimension> + (parallel::distributed::Triangulation &, + const dealii::Vector &, + const double, + const double); + \} + \} +\} +#endif +} diff --git a/deal.II/deal.II/source/distributed/solution_transfer.cc b/deal.II/deal.II/source/distributed/solution_transfer.cc new file mode 100644 index 0000000000..e4e19d65e7 --- /dev/null +++ b/deal.II/deal.II/source/distributed/solution_transfer.cc @@ -0,0 +1,273 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + +#include + +#ifdef DEAL_II_USE_P4EST + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +DEAL_II_NAMESPACE_OPEN + +namespace parallel +{ + namespace distributed + { + + namespace + { + template + void compress_vector_insert(VECTOR & vec) + { + vec.compress(); + } + +#ifdef DEAL_II_USE_TRILINOS + void compress_vector_insert(TrilinosWrappers::Vector & vec) + { + vec.compress(Insert); + } + + void compress_vector_insert(TrilinosWrappers::BlockVector & vec) + { + for (unsigned int i=0;i + SolutionTransfer::SolutionTransfer(const DH &dof) + : + dof_handler(&dof, typeid(*this).name()) + {} + + + + template + SolutionTransfer::~SolutionTransfer() + {} + + + + template + void + SolutionTransfer:: + prepare_for_coarsening_and_refinement (const std::vector &all_in) + { + Assert(all_in.size() > 0, ExcMessage("Please transfer atleast one vector!")); + input_vectors = all_in; + SolutionTransfer *ptr = this; + +//TODO: casting away constness is bad + parallel::distributed::Triangulation * tria + = (dynamic_cast*> + (const_cast*> + (&dof_handler->get_tria()))); + Assert (tria != 0, ExcInternalError()); + + offset + = tria->register_data_attach(static_cast + (get_data_size() * input_vectors.size()), + std_cxx1x::bind(&SolutionTransfer::pack_callback, + ptr, + _1, + _2, + _3)); + } + + + + template + void + SolutionTransfer:: + prepare_for_coarsening_and_refinement (const VECTOR &in) + { + std::vector all_in(1, &in); + prepare_for_coarsening_and_refinement(all_in); + } + + + + template + void + SolutionTransfer:: + interpolate (std::vector &all_out) + { + Assert(input_vectors.size()==all_out.size(), + ExcDimensionMismatch(input_vectors.size(), all_out.size()) ); + +//TODO: casting away constness is bad + parallel::distributed::Triangulation * tria + = (dynamic_cast*> + (const_cast*> + (&dof_handler->get_tria()))); + Assert (tria != 0, ExcInternalError()); + + tria->notify_ready_to_unpack(offset, + std_cxx1x::bind(&SolutionTransfer::unpack_callback, + this, + _1, + _2, + _3, + std_cxx1x::ref(all_out))); + + for (typename std::vector::iterator it=all_out.begin(); + it !=all_out.end(); + ++it) + compress_vector_insert(*(*it)); + + input_vectors.clear(); + } + + + + template + void + SolutionTransfer:: + interpolate (VECTOR &out) + { + std::vector all_out(1, &out); + interpolate(all_out); + } + + + + template + unsigned int + SolutionTransfer:: + get_data_size() const + { + return sizeof(double)* DoFTools::max_dofs_per_cell(*dof_handler); + } + + + template + void + SolutionTransfer:: + pack_callback(const typename Triangulation::cell_iterator & cell_, + const typename Triangulation::CellStatus /*status*/, + void* data) + { + double *data_store = reinterpret_cast(data); + + typename DH::cell_iterator cell(&dof_handler->get_tria(), cell_->level(), cell_->index(), dof_handler); + + const unsigned int dofs_per_cell=cell->get_fe().dofs_per_cell; + Vector dofvalues(dofs_per_cell); + for (typename std::vector::iterator it=input_vectors.begin(); + it !=input_vectors.end(); + ++it) + { + cell->get_interpolated_dof_values(*(*it), dofvalues); + std::memcpy(data_store, &dofvalues(0), sizeof(double)*dofs_per_cell); + data_store += dofs_per_cell; + } + } + + + template + void + SolutionTransfer:: + unpack_callback(const typename Triangulation::cell_iterator & cell_, + const typename Triangulation::CellStatus /*status*/, + const void* data, + std::vector &all_out) + { + typename DH::cell_iterator + cell(&dof_handler->get_tria(), cell_->level(), cell_->index(), dof_handler); + + const unsigned int dofs_per_cell=cell->get_fe().dofs_per_cell; + Vector dofvalues(dofs_per_cell); + const double *data_store = reinterpret_cast(data); + + for (typename std::vector::iterator it = all_out.begin(); + it != all_out.end(); + ++it) + { + std::memcpy(&dofvalues(0), data_store, sizeof(double)*dofs_per_cell); + cell->set_dof_values_by_interpolation(dofvalues, *(*it)); + data_store += dofs_per_cell; + } + } + + + } +} + + +// explicit instantiations + +namespace parallel +{ + namespace distributed + { +#if deal_II_dimension > 1 + template class SolutionTransfer, DoFHandler >; + + +#ifdef DEAL_II_USE_PETSC + template class SolutionTransfer >; + + template class SolutionTransfer >; + + template class SolutionTransfer >; + + template class SolutionTransfer >; + +#endif + +#ifdef DEAL_II_USE_TRILINOS + template class SolutionTransfer >; + + template class SolutionTransfer >; + + template class SolutionTransfer >; + + template class SolutionTransfer >; +#endif +#endif + } +} + +DEAL_II_NAMESPACE_CLOSE + +#endif diff --git a/deal.II/deal.II/source/distributed/tria.cc b/deal.II/deal.II/source/distributed/tria.cc new file mode 100644 index 0000000000..b99e8dfd63 --- /dev/null +++ b/deal.II/deal.II/source/distributed/tria.cc @@ -0,0 +1,3016 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef DEAL_II_USE_P4EST +# include +# include +# include +# include + +# include +# include +# include +# include +#endif + +#include +#include + +#include + +# include + + +DEAL_II_NAMESPACE_OPEN + + +#ifdef DEAL_II_USE_P4EST + +namespace internal +{ + namespace p4est + { + /** + * A structure whose explicit + * specializations contain + * pointers to the relevant + * p4est_* and p8est_* + * functions. Using this + * structure, for example by + * saying + * functions::quadrant_compare, + * we can write code in a + * dimension independent way, + * either calling + * p4est_quadrant_compare or + * p8est_quadrant_compare, + * depending on template + * argument. + */ + template struct functions; + + template <> struct functions<2> + { + static + int (&quadrant_compare) (const void *v1, const void *v2); + + static + void (&quadrant_childrenv) (const types<2>::quadrant * q, + types<2>::quadrant c[]); + + static + int (&quadrant_overlaps_tree) (types<2>::tree * tree, + const types<2>::quadrant * q); + + static + void (&quadrant_set_morton) (types<2>::quadrant * quadrant, + int level, + uint64_t id); + + static + int (&quadrant_is_equal) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2); + + static + int (&quadrant_is_sibling) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2); + + static + int (&quadrant_is_ancestor) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2); + + static + types<2>::connectivity * (&connectivity_new) (types<2>::topidx num_vertices, + types<2>::topidx num_trees, + types<2>::topidx num_corners, + types<2>::topidx num_vtt); + + static + void (&connectivity_destroy) (p4est_connectivity_t *connectivity); + + static + types<2>::forest * (&new_forest) (MPI_Comm mpicomm, + types<2>::connectivity * connectivity, + types<2>::locidx min_quadrants, + int min_level, + int fill_uniform, + size_t data_size, + p4est_init_t init_fn, + void *user_pointer); + + static + void (&destroy) (types<2>::forest * p4est); + + static + void (&refine) (types<2>::forest * p4est, + int refine_recursive, + p4est_refine_t refine_fn, + p4est_init_t init_fn); + + static + void (&coarsen) (types<2>::forest * p4est, + int coarsen_recursive, + p4est_coarsen_t coarsen_fn, + p4est_init_t init_fn); + + static + void (&balance) (types<2>::forest * p4est, + types<2>::balance_type btype, + p4est_init_t init_fn); + + static + void (&partition) (types<2>::forest * p4est, + int partition_for_coarsening, + p4est_weight_t weight_fn); + + static + void (&save) (const char *filename, + types<2>::forest * p4est, + int save_data); + + static + types<2>::forest * (&load) (const char *filename, + MPI_Comm mpicomm, + std::size_t data_size, + int load_data, + void *user_pointer, + types<2>::connectivity ** p4est); + + static + void (&connectivity_save) (const char *filename, + types<2>::connectivity * connectivity); + + static + types<2>::connectivity * (&connectivity_load) (const char *filename, + long *length); + + static + unsigned (&checksum) (types<2>::forest * p4est); + + static + void (&vtk_write_file) (types<2>::forest * p4est, + p4est_geometry_t*, + const char *baseName); + + static + types<2>::ghost* (&ghost_new) (types<2>::forest * p4est, + types<2>::balance_type btype); + + static + void (&ghost_destroy) (types<2>::ghost * ghost); + + static + void (&reset_data) (types<2>::forest * p4est, + size_t data_size, + p4est_init_t init_fn, + void *user_pointer); + + static + size_t (&forest_memory_used) (types<2>::forest * p4est); + + static + size_t (&connectivity_memory_used) (types<2>::connectivity * p4est); + }; + + int (&functions<2>::quadrant_compare) (const void *v1, const void *v2) + = p4est_quadrant_compare; + + void (&functions<2>::quadrant_childrenv) (const types<2>::quadrant * q, + types<2>::quadrant c[]) + = p4est_quadrant_childrenv; + + int (&functions<2>::quadrant_overlaps_tree) (types<2>::tree * tree, + const types<2>::quadrant * q) + = p4est_quadrant_overlaps_tree; + + void (&functions<2>::quadrant_set_morton) (types<2>::quadrant * quadrant, + int level, + uint64_t id) + = p4est_quadrant_set_morton; + + int (&functions<2>::quadrant_is_equal) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2) + = p4est_quadrant_is_equal; + + int (&functions<2>::quadrant_is_sibling) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2) + = p4est_quadrant_is_sibling; + + int (&functions<2>::quadrant_is_ancestor) (const types<2>::quadrant * q1, + const types<2>::quadrant * q2) + = p4est_quadrant_is_ancestor; + + types<2>::connectivity * (&functions<2>::connectivity_new) (types<2>::topidx num_vertices, + types<2>::topidx num_trees, + types<2>::topidx num_corners, + types<2>::topidx num_vtt) + = p4est_connectivity_new; + + void (&functions<2>::connectivity_destroy) (p4est_connectivity_t *connectivity) + = p4est_connectivity_destroy; + + types<2>::forest * (&functions<2>::new_forest) (MPI_Comm mpicomm, + types<2>::connectivity * connectivity, + types<2>::locidx min_quadrants, + int min_level, + int fill_uniform, + size_t data_size, + p4est_init_t init_fn, + void *user_pointer) + = p4est_new_ext; + + void (&functions<2>::destroy) (types<2>::forest * p4est) + = p4est_destroy; + + void (&functions<2>::refine) (types<2>::forest * p4est, + int refine_recursive, + p4est_refine_t refine_fn, + p4est_init_t init_fn) + = p4est_refine; + + void (&functions<2>::coarsen) (types<2>::forest * p4est, + int coarsen_recursive, + p4est_coarsen_t coarsen_fn, + p4est_init_t init_fn) + = p4est_coarsen; + + void (&functions<2>::balance) (types<2>::forest * p4est, + types<2>::balance_type btype, + p4est_init_t init_fn) + = p4est_balance; + + void (&functions<2>::partition) (types<2>::forest * p4est, + int partition_for_coarsening, + p4est_weight_t weight_fn) + = p4est_partition_ext; + + void (&functions<2>::save) (const char *filename, + types<2>::forest * p4est, + int save_data) + = p4est_save; + + types<2>::forest * + (&functions<2>::load) (const char *filename, + MPI_Comm mpicomm, + std::size_t data_size, + int load_data, + void *user_pointer, + types<2>::connectivity ** p4est) + = p4est_load; + + void (&functions<2>::connectivity_save) (const char *filename, + types<2>::connectivity *connectivity) + = p4est_connectivity_save; + + types<2>::connectivity * + (&functions<2>::connectivity_load) (const char *filename, + long *length) + = p4est_connectivity_load; + + unsigned (&functions<2>::checksum) (types<2>::forest * p4est) + = p4est_checksum; + + void (&functions<2>::vtk_write_file) (types<2>::forest * p4est, + p4est_geometry_t*, + const char *baseName) + = p4est_vtk_write_file; + + types<2>::ghost* (&functions<2>::ghost_new) (types<2>::forest * p4est, + types<2>::balance_type btype) + = p4est_ghost_new; + + void (&functions<2>::ghost_destroy) (types<2>::ghost * ghost) + = p4est_ghost_destroy; + + void (&functions<2>::reset_data) (types<2>::forest * p4est, + size_t data_size, + p4est_init_t init_fn, + void *user_pointer) + = p4est_reset_data; + + size_t (&functions<2>::forest_memory_used) (types<2>::forest * p4est) + = p4est_memory_used; + + size_t (&functions<2>::connectivity_memory_used) (types<2>::connectivity * p4est) + = p4est_connectivity_memory_used; + + template <> struct functions<3> + { + static + int (&quadrant_compare) (const void *v1, const void *v2); + + static + void (&quadrant_childrenv) (const types<3>::quadrant * q, + types<3>::quadrant c[]); + + static + int (&quadrant_overlaps_tree) (types<3>::tree * tree, + const types<3>::quadrant * q); + + static + void (&quadrant_set_morton) (types<3>::quadrant * quadrant, + int level, + uint64_t id); + + static + int (&quadrant_is_equal) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2); + + static + int (&quadrant_is_sibling) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2); + + static + int (&quadrant_is_ancestor) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2); + + static + types<3>::connectivity * (&connectivity_new) (types<3>::topidx num_vertices, + types<3>::topidx num_trees, + types<3>::topidx num_edges, + types<3>::topidx num_ett, + types<3>::topidx num_corners, + types<3>::topidx num_ctt); + + static + void (&connectivity_destroy) (p8est_connectivity_t *connectivity); + + static + types<3>::forest * (&new_forest) (MPI_Comm mpicomm, + types<3>::connectivity * connectivity, + types<3>::locidx min_quadrants, + int min_level, + int fill_uniform, + size_t data_size, + p8est_init_t init_fn, + void *user_pointer); + + static + void (&destroy) (types<3>::forest * p8est); + + static + void (&refine) (types<3>::forest * p8est, + int refine_recursive, + p8est_refine_t refine_fn, + p8est_init_t init_fn); + + static + void (&coarsen) (types<3>::forest * p8est, + int coarsen_recursive, + p8est_coarsen_t coarsen_fn, + p8est_init_t init_fn); + + static + void (&balance) (types<3>::forest * p8est, + types<3>::balance_type btype, + p8est_init_t init_fn); + + static + void (&partition) (types<3>::forest * p8est, + int partition_for_coarsening, + p8est_weight_t weight_fn); + + static + void (&save) (const char *filename, + types<3>::forest * p4est, + int save_data); + + static + types<3>::forest * (&load) (const char *filename, + MPI_Comm mpicomm, + std::size_t data_size, + int load_data, + void *user_pointer, + types<3>::connectivity ** p4est); + + static + void (&connectivity_save) (const char *filename, + types<3>::connectivity * connectivity); + + static + types<3>::connectivity * (&connectivity_load) (const char *filename, + long *length); + + static + unsigned (&checksum) (types<3>::forest * p8est); + + static + void (&vtk_write_file) (types<3>::forest * p8est, + p8est_geometry_t*, + const char *baseName); + + static + types<3>::ghost* (&ghost_new) (types<3>::forest * p4est, + types<3>::balance_type btype); + + static + void (&ghost_destroy) (types<3>::ghost * ghost); + + static + void (&reset_data) (types<3>::forest * p4est, + size_t data_size, + p8est_init_t init_fn, + void *user_pointer); + + static + size_t (&forest_memory_used) (types<3>::forest * p4est); + + static + size_t (&connectivity_memory_used) (types<3>::connectivity * p4est); + }; + + int (&functions<3>::quadrant_compare) (const void *v1, const void *v2) + = p8est_quadrant_compare; + + void (&functions<3>::quadrant_childrenv) (const types<3>::quadrant * q, + types<3>::quadrant c[]) + = p8est_quadrant_childrenv; + + int (&functions<3>::quadrant_overlaps_tree) (types<3>::tree * tree, + const types<3>::quadrant * q) + = p8est_quadrant_overlaps_tree; + + void (&functions<3>::quadrant_set_morton) (types<3>::quadrant * quadrant, + int level, + uint64_t id) + = p8est_quadrant_set_morton; + + int (&functions<3>::quadrant_is_equal) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2) + = p8est_quadrant_is_equal; + + int (&functions<3>::quadrant_is_sibling) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2) + = p8est_quadrant_is_sibling; + + int (&functions<3>::quadrant_is_ancestor) (const types<3>::quadrant * q1, + const types<3>::quadrant * q2) + = p8est_quadrant_is_ancestor; + + types<3>::connectivity * (&functions<3>::connectivity_new) (types<3>::topidx num_vertices, + types<3>::topidx num_trees, + types<3>::topidx num_edges, + types<3>::topidx num_ett, + types<3>::topidx num_corners, + types<3>::topidx num_ctt) + = p8est_connectivity_new; + + void (&functions<3>::connectivity_destroy) (p8est_connectivity_t *connectivity) + = p8est_connectivity_destroy; + + types<3>::forest * (&functions<3>::new_forest) (MPI_Comm mpicomm, + types<3>::connectivity * connectivity, + types<3>::locidx min_quadrants, + int min_level, + int fill_uniform, + size_t data_size, + p8est_init_t init_fn, + void *user_pointer) + = p8est_new_ext; + + void (&functions<3>::destroy) (types<3>::forest * p8est) + = p8est_destroy; + + void (&functions<3>::refine) (types<3>::forest * p8est, + int refine_recursive, + p8est_refine_t refine_fn, + p8est_init_t init_fn) + = p8est_refine; + + void (&functions<3>::coarsen) (types<3>::forest * p8est, + int coarsen_recursive, + p8est_coarsen_t coarsen_fn, + p8est_init_t init_fn) + = p8est_coarsen; + + void (&functions<3>::balance) (types<3>::forest * p8est, + types<3>::balance_type btype, + p8est_init_t init_fn) + = p8est_balance; + + void (&functions<3>::partition) (types<3>::forest * p8est, + int partition_for_coarsening, + p8est_weight_t weight_fn) + = p8est_partition_ext; + + void (&functions<3>::save) (const char *filename, + types<3>::forest * p4est, + int save_data) + = p8est_save; + + types<3>::forest * + (&functions<3>::load) (const char *filename, + MPI_Comm mpicomm, + std::size_t data_size, + int load_data, + void *user_pointer, + types<3>::connectivity ** p4est) + = p8est_load; + + void (&functions<3>::connectivity_save) (const char *filename, + types<3>::connectivity *connectivity) + = p8est_connectivity_save; + + types<3>::connectivity * + (&functions<3>::connectivity_load) (const char *filename, + long *length) + = p8est_connectivity_load; + + unsigned (&functions<3>::checksum) (types<3>::forest * p8est) + = p8est_checksum; + + void (&functions<3>::vtk_write_file) (types<3>::forest * p8est, + p8est_geometry_t*, + const char *baseName) + = p8est_vtk_write_file; + + types<3>::ghost* (&functions<3>::ghost_new) (types<3>::forest * p4est, + types<3>::balance_type btype) + = p8est_ghost_new; + + void (&functions<3>::ghost_destroy) (types<3>::ghost * ghost) + = p8est_ghost_destroy; + + void (&functions<3>::reset_data) (types<3>::forest * p4est, + size_t data_size, + p8est_init_t init_fn, + void *user_pointer) + = p8est_reset_data; + + size_t (&functions<3>::forest_memory_used) (types<3>::forest * p4est) + = p8est_memory_used; + + size_t (&functions<3>::connectivity_memory_used) (types<3>::connectivity * p4est) + = p8est_connectivity_memory_used; + + + + + template + void + init_quadrant_children + (const typename types::quadrant & p4est_cell, + typename types::quadrant (&p4est_children)[GeometryInfo::max_children_per_cell]) + { + + for (unsigned int c=0; + c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_children[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_children[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + + functions::quadrant_childrenv (&p4est_cell, + p4est_children); + + } + + + template + void + init_coarse_quadrant(typename types::quadrant & quad) + { + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&quad); + break; + case 3: + P8EST_QUADRANT_INIT(&quad); + break; + default: + Assert (false, ExcNotImplemented()); + } + functions::quadrant_set_morton (&quad, + /*level=*/0, + /*index=*/0); + } + + + template + bool + quadrant_is_equal (const typename types::quadrant & q1, + const typename types::quadrant & q2) + { + return functions::quadrant_is_equal(&q1, &q2); + } + + + + template + bool + quadrant_is_ancestor (const typename types::quadrant & q1, + const typename types::quadrant & q2) + { + return functions::quadrant_is_ancestor(&q1, &q2); + } + } +} + + +namespace +{ + template + void + get_vertex_to_cell_mappings (const Triangulation &triangulation, + std::vector &vertex_touch_count, + std::vector::active_cell_iterator,unsigned int> > > + & vertex_to_cell) + { + vertex_touch_count.resize (triangulation.n_vertices()); + vertex_to_cell.resize (triangulation.n_vertices()); + + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell) + for (unsigned int v=0; v::vertices_per_cell; ++v) + { + ++vertex_touch_count[cell->vertex_index(v)]; + vertex_to_cell[cell->vertex_index(v)] + .push_back (std::make_pair (cell, v)); + } + } + + + + template + void + get_edge_to_cell_mappings (const Triangulation &triangulation, + std::vector &edge_touch_count, + std::vector::active_cell_iterator,unsigned int> > > + & edge_to_cell) + { + Assert (triangulation.n_levels() == 1, ExcInternalError()); + + edge_touch_count.resize (triangulation.n_active_lines()); + edge_to_cell.resize (triangulation.n_active_lines()); + + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell) + for (unsigned int l=0; l::lines_per_cell; ++l) + { + ++edge_touch_count[cell->line(l)->index()]; + edge_to_cell[cell->line(l)->index()] + .push_back (std::make_pair (cell, l)); + } + } + + + + /** + * Set all vertex and cell + * related information in the + * p4est connectivity structure. + */ + template + void + set_vertex_and_cell_info (const Triangulation &triangulation, + const std::vector &vertex_touch_count, + const std::vector::active_cell_iterator,unsigned int> > > + & vertex_to_cell, + const std::vector &coarse_cell_to_p4est_tree_permutation, + const bool set_vertex_info, + typename internal::p4est::types::connectivity *connectivity) + { + // copy the vertices into the + // connectivity structure. the + // triangulation exports the array of + // vertices, but some of the entries are + // sometimes unused; this shouldn't be + // the case for a newly created + // triangulation, but make sure + // + // note that p4est stores coordinates as + // a triplet of values even in 2d + Assert (triangulation.get_used_vertices().size() == + triangulation.get_vertices().size(), + ExcInternalError()); + Assert (std::find (triangulation.get_used_vertices().begin(), + triangulation.get_used_vertices().end(), + false) + == triangulation.get_used_vertices().end(), + ExcInternalError()); + if (set_vertex_info == true) + for (unsigned int v=0; vvertices[3*v ] = triangulation.get_vertices()[v][0]; + connectivity->vertices[3*v+1] = triangulation.get_vertices()[v][1]; + connectivity->vertices[3*v+2] = (spacedim == 2 ? + 0 + : + triangulation.get_vertices()[v][2]); + } + + // next store the tree_to_vertex indices + // (each tree is here only a single cell + // in the coarse mesh). p4est requires + // vertex numbering in clockwise + // orientation + // + // while we're at it, also copy the + // neighborship information between cells + typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(), + endc = triangulation.end(); + for (; cell != endc; ++cell) + { + const unsigned int + index = coarse_cell_to_p4est_tree_permutation[cell->index()]; + + for (unsigned int v=0; v::vertices_per_cell; ++v) + { + if (set_vertex_info == true) + connectivity->tree_to_vertex[index*GeometryInfo::vertices_per_cell+v] = cell->vertex_index(v); + connectivity->tree_to_corner[index*GeometryInfo::vertices_per_cell+v] = cell->vertex_index(v); + } + + // neighborship information. if a + // cell is at a boundary, then enter + // the index of the cell itself here + for (unsigned int f=0; f::faces_per_cell; ++f) + if (cell->face(f)->at_boundary() == false) + connectivity->tree_to_tree[index*GeometryInfo::faces_per_cell + f] + = coarse_cell_to_p4est_tree_permutation[cell->neighbor(f)->index()]; + else + connectivity->tree_to_tree[index*GeometryInfo::faces_per_cell + f] + = coarse_cell_to_p4est_tree_permutation[cell->index()]; + + // fill tree_to_face, which is + // essentially neighbor_to_neighbor; + // however, we have to remap the + // resulting face number as well + for (unsigned int f=0; f::faces_per_cell; ++f) + if (cell->face(f)->at_boundary() == false) + { + switch (dim) + { + case 2: + { + connectivity->tree_to_face[index*GeometryInfo::faces_per_cell + f] + = cell->neighbor_of_neighbor (f); + break; + } + + case 3: + { + /* + * The values for + * tree_to_face are in + * 0..23 where ttf % 6 + * gives the face + * number and ttf / 6 + * the face orientation + * code. The + * orientation is + * determined as + * follows. Let + * my_face and + * other_face be the + * two face numbers of + * the connecting trees + * in 0..5. Then the + * first face vertex of + * the lower of my_face + * and other_face + * connects to a face + * vertex numbered 0..3 + * in the higher of + * my_face and + * other_face. The + * face orientation is + * defined as this + * number. If my_face + * == other_face, + * treating either of + * both faces as the + * lower one leads to + * the same result. + */ + + connectivity->tree_to_face[index*6 + f] + = cell->neighbor_of_neighbor (f); + + unsigned int face_idx_list[2] = + {f, cell->neighbor_of_neighbor (f)}; + typename Triangulation::active_cell_iterator + cell_list[2] = {cell, cell->neighbor(f)}; + unsigned int smaller_idx = 0; + + if (f>cell->neighbor_of_neighbor (f)) + smaller_idx = 1; + + unsigned larger_idx = (smaller_idx+1) % 2; + //smaller = *_list[smaller_idx] + //larger = *_list[larger_idx] + + unsigned int v = 0; + + // global vertex index of + // vertex 0 on face of + // cell with smaller + // local face index + unsigned int g_idx = + cell_list[smaller_idx]->vertex_index( + GeometryInfo::face_to_cell_vertices( + face_idx_list[smaller_idx], + 0, + cell_list[smaller_idx]->face_orientation(face_idx_list[smaller_idx]), + cell_list[smaller_idx]->face_flip(face_idx_list[smaller_idx]), + cell_list[smaller_idx]->face_rotation(face_idx_list[smaller_idx])) + ); + + // loop over vertices on + // face from other cell + // and compare global + // vertex numbers + for (unsigned int i=0; i::vertices_per_face; ++i) + { + unsigned int idx + = + cell_list[larger_idx]->vertex_index( + GeometryInfo::face_to_cell_vertices( + face_idx_list[larger_idx], + i) + ); + + if (idx==g_idx) + { + v = i; + break; + } + } + + connectivity->tree_to_face[index*6 + f] += 6*v; + break; + } + + default: + Assert (false, ExcNotImplemented()); + } + } + else + connectivity->tree_to_face[index*GeometryInfo::faces_per_cell + f] = f; + } + + // now fill the vertex information + connectivity->ctt_offset[0] = 0; + std::partial_sum (vertex_touch_count.begin(), + vertex_touch_count.end(), + &connectivity->ctt_offset[1]); + + const typename internal::p4est::types::locidx + num_vtt = std::accumulate (vertex_touch_count.begin(), + vertex_touch_count.end(), + 0); + Assert (connectivity->ctt_offset[triangulation.n_vertices()] == + num_vtt, + ExcInternalError()); + + for (unsigned int v=0; v::active_cell_iterator, + unsigned int> >::const_iterator + p = vertex_to_cell[v].begin(); + for (unsigned int c=0; ccorner_to_tree[connectivity->ctt_offset[v]+c] + = coarse_cell_to_p4est_tree_permutation[p->first->index()]; + connectivity->corner_to_corner[connectivity->ctt_offset[v]+c] + = p->second; + } + } + } + + + + template + bool + tree_exists_locally (const typename internal::p4est::types::forest *parallel_forest, + const typename internal::p4est::types::topidx coarse_grid_cell) + { + Assert (coarse_grid_cell < parallel_forest->connectivity->num_trees, + ExcInternalError()); + return ((coarse_grid_cell >= parallel_forest->first_local_tree) + && + (coarse_grid_cell <= parallel_forest->last_local_tree)); + } + + + template + void + delete_all_children_and_self (const typename Triangulation::cell_iterator &cell) + { + if (cell->has_children()) + for (unsigned int c=0; cn_children(); ++c) + delete_all_children_and_self (cell->child(c)); + else + cell->set_coarsen_flag (); + } + + + + template + void + delete_all_children (const typename Triangulation::cell_iterator &cell) + { + if (cell->has_children()) + for (unsigned int c=0; cn_children(); ++c) + delete_all_children_and_self (cell->child(c)); + } + + + + template + void + match_tree_recursively (const typename internal::p4est::types::tree &tree, + const typename Triangulation::cell_iterator &dealii_cell, + const typename internal::p4est::types::quadrant &p4est_cell, + const typename internal::p4est::types::forest &forest, + const types::subdomain_id_t my_subdomain) + { + // check if this cell exists in + // the local p4est cell + if (sc_array_bsearch(const_cast(&tree.quadrants), + &p4est_cell, + internal::p4est::functions::quadrant_compare) + != -1) + { + // yes, cell found in local part of p4est + delete_all_children (dealii_cell); + dealii_cell->set_subdomain_id(my_subdomain); + } + else + { + // no, cell not found in + // local part of + // p4est. this means that + // the local part is more + // refined than the current + // cell. if this cell has + // no children of its own, + // we need to refine it, + // and if it does already + // have children then loop + // over all children and + // see if they are locally + // available as well + if (dealii_cell->has_children () == false) + dealii_cell->set_refine_flag (); + else + { + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; + c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + + internal::p4est::functions:: + quadrant_childrenv (&p4est_cell, + p4est_child); + + for (unsigned int c=0; + c::max_children_per_cell; ++c) + if (internal::p4est::functions:: + quadrant_overlaps_tree (const_cast::tree*>(&tree), + &p4est_child[c]) + == false) + { + // no, this child + // is locally not + // available in + // the p4est. + // delete all + // its children + // but, because this + // may not be successfull, + // make sure to mark all + // children recursively as + // not local. + delete_all_children (dealii_cell->child(c)); + dealii_cell->child(c) + ->recursively_set_subdomain_id(types::artificial_subdomain_id); + } + else + { + // at least some + // part of the + // tree rooted in + // this child is + // locally + // available + match_tree_recursively (tree, + dealii_cell->child(c), + p4est_child[c], + forest, + my_subdomain); + } + } + } + } + + + template + void + match_quadrant_recursively (const typename internal::p4est::types::quadrant &this_quadrant, + const typename Triangulation::cell_iterator &dealii_cell, + const typename internal::p4est::types::quadrant &ghost_quadrant, + const typename internal::p4est::types::forest &forest, + unsigned int ghost_owner) + { + if (internal::p4est::functions::quadrant_is_equal(&this_quadrant, &ghost_quadrant)) + { + // this is the ghostcell + dealii_cell->set_subdomain_id(ghost_owner); + + if (dealii_cell->has_children()) + delete_all_children (dealii_cell); + else + dealii_cell->clear_coarsen_flag(); + return; + } + + if (! internal::p4est::functions::quadrant_is_ancestor ( &this_quadrant, &ghost_quadrant)) + { + return; + } + + if (dealii_cell->has_children () == false) + { + dealii_cell->clear_coarsen_flag(); + dealii_cell->set_refine_flag (); + return; + } + + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; + c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + + internal::p4est::functions:: + quadrant_childrenv (&this_quadrant, p4est_child); + + for (unsigned int c=0; + c::max_children_per_cell; ++c) + + { + match_quadrant_recursively (p4est_child[c], dealii_cell->child(c), ghost_quadrant, forest, ghost_owner); + } + + + } + + + + template + void + attach_mesh_data_recursively (const typename internal::p4est::types::tree &tree, + const typename Triangulation::cell_iterator &dealii_cell, + const typename internal::p4est::types::quadrant &p4est_cell, + const typename std::list::cell_iterator, + typename parallel::distributed::Triangulation::CellStatus, + void*) + > > > &attached_data_pack_callbacks) + { + typedef std::list::cell_iterator, + typename parallel::distributed::Triangulation::CellStatus, + void*) + > > > callback_list_t; + + int idx = sc_array_bsearch(const_cast(&tree.quadrants), + &p4est_cell, + internal::p4est::functions::quadrant_compare); + + if (idx == -1 && (internal::p4est::functions:: + quadrant_overlaps_tree (const_cast::tree*>(&tree), + &p4est_cell) + == false)) + return; //this quadrant and none of it's childs belongs to us. + + bool p4est_has_children = (idx == -1); + + if (p4est_has_children && dealii_cell->has_children()) + { + //recurse further + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + internal::p4est::functions:: + quadrant_childrenv (&p4est_cell, p4est_child); + + for (unsigned int c=0; + c::max_children_per_cell; ++c) + { + attach_mesh_data_recursively (tree, + dealii_cell->child(c), + p4est_child[c], + attached_data_pack_callbacks); + } + } + else if (!p4est_has_children && !dealii_cell->has_children()) + { + //this active cell didn't change + typename internal::p4est::types::quadrant *q; + q = static_cast::quadrant*> ( + sc_array_index (const_cast(&tree.quadrants), idx) + ); + *static_cast::CellStatus*>(q->p.user_data) = parallel::distributed::Triangulation::CELL_PERSIST; + + for(typename callback_list_t::const_iterator it = attached_data_pack_callbacks.begin(); + it != attached_data_pack_callbacks.end(); + ++it) + { + void * ptr = static_cast(q->p.user_data) + (*it).first; //add offset + ((*it).second)(dealii_cell, + parallel::distributed::Triangulation::CELL_PERSIST, + ptr); + } + } + else if (p4est_has_children) + { + //this cell got refined + + //attach to the first child, because + // we can only attach to active + // quadrants + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + internal::p4est::functions:: + quadrant_childrenv (&p4est_cell, p4est_child); + int child0_idx = sc_array_bsearch(const_cast(&tree.quadrants), + &p4est_child[0], + internal::p4est::functions::quadrant_compare); + Assert(child0_idx != -1, ExcMessage("the first child should exist as an active quadrant!")); + + typename internal::p4est::types::quadrant *q; + q = static_cast::quadrant*> ( + sc_array_index (const_cast(&tree.quadrants), child0_idx) + ); + *static_cast::CellStatus*>(q->p.user_data) = parallel::distributed::Triangulation::CELL_REFINE; + + for(typename callback_list_t::const_iterator it = attached_data_pack_callbacks.begin(); + it != attached_data_pack_callbacks.end(); + ++it) + { + void * ptr = static_cast(q->p.user_data) + (*it).first; //add offset + + ((*it).second)(dealii_cell, + parallel::distributed::Triangulation::CELL_REFINE, + ptr); + } + + //mark other childs as invalid, so that unpack only happens once + for (unsigned int i=1;i::max_children_per_cell; ++i) + { + int child_idx = sc_array_bsearch(const_cast(&tree.quadrants), + &p4est_child[i], + internal::p4est::functions::quadrant_compare); + q = static_cast::quadrant*> ( + sc_array_index (const_cast(&tree.quadrants), child_idx) + ); + *static_cast::CellStatus*>(q->p.user_data) = parallel::distributed::Triangulation::CELL_INVALID; + } + + + } + else + { + //it's children got coarsened into + //this cell + typename internal::p4est::types::quadrant *q; + q = static_cast::quadrant*> ( + sc_array_index (const_cast(&tree.quadrants), idx) + ); + *static_cast::CellStatus*>(q->p.user_data) = parallel::distributed::Triangulation::CELL_COARSEN; + + for(typename callback_list_t::const_iterator it = attached_data_pack_callbacks.begin(); + it != attached_data_pack_callbacks.end(); + ++it) + { + void * ptr = static_cast(q->p.user_data) + (*it).first; //add offset + ((*it).second)(dealii_cell, + parallel::distributed::Triangulation::CELL_COARSEN, + ptr); + } + } + } + + + template + void + post_mesh_data_recursively (const typename internal::p4est::types::tree &tree, + const typename Triangulation::cell_iterator &dealii_cell, + const typename Triangulation::cell_iterator &parent_cell, + const typename internal::p4est::types::quadrant &p4est_cell, + const unsigned int offset, + const typename std_cxx1x::function< + void(typename parallel::distributed::Triangulation::cell_iterator, typename parallel::distributed::Triangulation::CellStatus, void*) + > &unpack_callback) + { + int idx = sc_array_bsearch(const_cast(&tree.quadrants), + &p4est_cell, + internal::p4est::functions::quadrant_compare); + if (idx == -1 && (internal::p4est::functions:: + quadrant_overlaps_tree (const_cast::tree*>(&tree), + &p4est_cell) + == false)) + // this quadrant and none of + // it's children belong to us. + return; + + + const bool p4est_has_children = (idx == -1); + if (p4est_has_children) + { + Assert(dealii_cell->has_children(), ExcInternalError()); + + //recurse further + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + + internal::p4est::functions:: + quadrant_childrenv (&p4est_cell, p4est_child); + + for (unsigned int c=0; + c::max_children_per_cell; ++c) + { + post_mesh_data_recursively (tree, + dealii_cell->child(c), + dealii_cell, + p4est_child[c], + offset, + unpack_callback); + } + } + else + { + Assert(! dealii_cell->has_children(), ExcInternalError()); + + typename internal::p4est::types::quadrant *q; + q = static_cast::quadrant*> ( + sc_array_index (const_cast(&tree.quadrants), idx) + ); + + void * ptr = static_cast(q->p.user_data) + offset; + typename parallel::distributed::Triangulation::CellStatus + status = * static_cast< + typename parallel::distributed::Triangulation::CellStatus* + >(q->p.user_data); + switch (status) + { + case parallel::distributed::Triangulation::CELL_PERSIST: + { + unpack_callback(dealii_cell, status, ptr); + break; + } + case parallel::distributed::Triangulation::CELL_REFINE: + { + unpack_callback(parent_cell, status, ptr); + break; + } + case parallel::distributed::Triangulation::CELL_COARSEN: + { + unpack_callback(dealii_cell, status, ptr); + break; + } + case parallel::distributed::Triangulation::CELL_INVALID: + { + break; + } + default: + throw ExcInternalError(); + } + } + } + + + + /** + * A data structure that we use to store + * which cells (indicated by + * internal::p4est::types::quadrant objects) shall be + * refined and which shall be coarsened. + */ + template + class RefineAndCoarsenList + { + public: + RefineAndCoarsenList (const Triangulation &triangulation, + const std::vector &p4est_tree_to_coarse_cell_permutation, + const types::subdomain_id_t my_subdomain, + typename internal::p4est::types::forest &forest); + + /** + * A callback function that we + * pass to the p4est data + * structures when a forest is + * to be refined. The p4est + * functions call it back with + * a tree (the index of the + * tree that grows out of a + * given coarse cell) and a + * refinement path from that + * coarse cell to a + * terminal/leaf cell. The + * function returns whether the + * corresponding cell in the + * deal.II triangulation has + * the refined flag set. + */ + static + int + refine_callback (typename internal::p4est::types::forest *forest, + typename internal::p4est::types::topidx coarse_cell_index, + typename internal::p4est::types::quadrant *quadrant); + + /** + * Same as the refine_callback + * function, but return whether + * all four of the given + * children of a non-terminal + * cell are to be coarsened + * away. + */ + static + int + coarsen_callback (typename internal::p4est::types::forest *forest, + typename internal::p4est::types::topidx coarse_cell_index, + typename internal::p4est::types::quadrant *children[]); + + bool pointers_are_at_end () const; + + private: + std::vector::quadrant> refine_list; + typename std::vector::quadrant>::const_iterator current_refine_pointer; + + std::vector::quadrant> coarsen_list; + typename std::vector::quadrant>::const_iterator current_coarsen_pointer; + + typename internal::p4est::types::forest forest; + + void build_lists (const typename Triangulation::cell_iterator &cell, + const typename internal::p4est::types::quadrant &p4est_cell, + const unsigned int myid); + }; + + + + template + bool + RefineAndCoarsenList:: + pointers_are_at_end () const + { + return ((current_refine_pointer == refine_list.end()) + && + (current_coarsen_pointer == coarsen_list.end())); + } + + + + template + RefineAndCoarsenList:: + RefineAndCoarsenList (const Triangulation &triangulation, + const std::vector &p4est_tree_to_coarse_cell_permutation, + const types::subdomain_id_t my_subdomain, + typename internal::p4est::types::forest &forest) + : + forest(forest) + { + // count how many flags are set and + // allocate that much memory + unsigned int n_refine_flags = 0, + n_coarsen_flags = 0; + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell) + { + //skip cells that are not local + if (cell->subdomain_id() != my_subdomain) + continue; + + if (cell->refine_flag_set()) + ++n_refine_flags; + else if (cell->coarsen_flag_set()) + ++n_coarsen_flags; + } + + refine_list.reserve (n_refine_flags); + coarsen_list.reserve (n_coarsen_flags); + + + // now build the lists of cells that + // are flagged. note that p4est will + // traverse its cells in the order in + // which trees appear in the + // forest. this order is not the same + // as the order of coarse cells in the + // deal.II Triangulation because we + // have translated everything by the + // coarse_cell_to_p4est_tree_permutation + // permutation. in order to make sure + // that the output array is already in + // the correct order, traverse our + // coarse cells in the same order in + // which p4est will: + for (unsigned int c=0; c::cell_iterator + cell (&triangulation, 0, coarse_cell_index); + + typename internal::p4est::types::quadrant p4est_cell; + internal::p4est::functions:: + quadrant_set_morton (&p4est_cell, + /*level=*/0, + /*index=*/0); + p4est_cell.p.which_tree = c; + build_lists (cell, p4est_cell, my_subdomain); + } + + + Assert(refine_list.size() == n_refine_flags, + ExcInternalError()); + Assert(coarsen_list.size() == n_coarsen_flags, + ExcInternalError()); + + // make sure that our ordering in fact + // worked + for (unsigned int i=1; i= + refine_list[i-1].p.which_tree, + ExcInternalError()); + for (unsigned int i=1; i= + coarsen_list[i-1].p.which_tree, + ExcInternalError()); + + current_refine_pointer = refine_list.begin(); + current_coarsen_pointer = coarsen_list.begin(); + } + + + + template + void + RefineAndCoarsenList:: + build_lists (const typename Triangulation::cell_iterator &cell, + const typename internal::p4est::types::quadrant &p4est_cell, + const types::subdomain_id_t my_subdomain) + { + if (!cell->has_children()) + { + if (cell->subdomain_id() == my_subdomain) + { + if (cell->refine_flag_set()) + refine_list.push_back (p4est_cell); + else if (cell->coarsen_flag_set()) + coarsen_list.push_back (p4est_cell); + } + } + else + { + typename internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + for (unsigned int c=0; c::max_children_per_cell; ++c) + switch (dim) + { + case 2: + P4EST_QUADRANT_INIT(&p4est_child[c]); + break; + case 3: + P8EST_QUADRANT_INIT(&p4est_child[c]); + break; + default: + Assert (false, ExcNotImplemented()); + } + internal::p4est::functions:: + quadrant_childrenv (&p4est_cell, + p4est_child); + for (unsigned int c=0; + c::max_children_per_cell; ++c) + { + p4est_child[c].p.which_tree = p4est_cell.p.which_tree; + build_lists (cell->child(c), + p4est_child[c], + my_subdomain); + } + } + } + + + template + int + RefineAndCoarsenList:: + refine_callback (typename internal::p4est::types::forest *forest, + typename internal::p4est::types::topidx coarse_cell_index, + typename internal::p4est::types::quadrant *quadrant) + { + RefineAndCoarsenList *this_object + = reinterpret_cast*>(forest->user_pointer); + + // if there are no more cells in our + // list the current cell can't be + // flagged for refinement + if (this_object->current_refine_pointer == this_object->refine_list.end()) + return false; + + Assert (coarse_cell_index <= + this_object->current_refine_pointer->p.which_tree, + ExcInternalError()); + + // if p4est hasn't yet reached the tree + // of the next flagged cell the current + // cell can't be flagged for refinement + if (coarse_cell_index < + this_object->current_refine_pointer->p.which_tree) + return false; + + // now we're in the right tree in the + // forest + Assert (coarse_cell_index <= + this_object->current_refine_pointer->p.which_tree, + ExcInternalError()); + + // make sure that the p4est loop over + // cells hasn't gotten ahead of our own + // pointer + Assert (internal::p4est::functions:: + quadrant_compare (quadrant, + &*this_object->current_refine_pointer) <= 0, + ExcInternalError()); + + // now, if the p4est cell is one in the + // list, it is supposed to be refined + if (internal::p4est::functions:: + quadrant_is_equal (quadrant, &*this_object->current_refine_pointer)) + { + ++this_object->current_refine_pointer; + return true; + } + + // p4est cell is not in list + return false; + } + + + + template + int + RefineAndCoarsenList:: + coarsen_callback (typename internal::p4est::types::forest *forest, + typename internal::p4est::types::topidx coarse_cell_index, + typename internal::p4est::types::quadrant *children[]) + { + RefineAndCoarsenList *this_object + = reinterpret_cast*>(forest->user_pointer); + + // if there are no more cells in our + // list the current cell can't be + // flagged for coarsening + if (this_object->current_coarsen_pointer == + this_object->coarsen_list.end()) + return false; + + Assert (coarse_cell_index <= + this_object->current_coarsen_pointer->p.which_tree, + ExcInternalError()); + + // if p4est hasn't yet reached the tree + // of the next flagged cell the current + // cell can't be flagged for coarsening + if (coarse_cell_index < + this_object->current_coarsen_pointer->p.which_tree) + return false; + + // now we're in the right tree in the + // forest + Assert (coarse_cell_index <= + this_object->current_coarsen_pointer->p.which_tree, + ExcInternalError()); + + // make sure that the p4est loop over + // cells hasn't gotten ahead of our own + // pointer + Assert (internal::p4est::functions:: + quadrant_compare (children[0], + &*this_object->current_coarsen_pointer) <= 0, + ExcInternalError()); + + // now, if the p4est cell is one in the + // list, it is supposed to be coarsened + if (internal::p4est::functions:: + quadrant_is_equal (children[0], + &*this_object->current_coarsen_pointer)) + { + // move current pointer one up + ++this_object->current_coarsen_pointer; + + // note that the next 3 cells in + // our list need to correspond to + // the other siblings of the cell + // we have just found + for (unsigned int c=1; c::max_children_per_cell; ++c) + { + Assert (internal::p4est::functions:: + quadrant_is_equal (children[c], + &*this_object->current_coarsen_pointer), + ExcInternalError()); + ++this_object->current_coarsen_pointer; + } + + return true; + } + + // p4est cell is not in list + return false; + } +} + + +// initialize p4est +namespace internal +{ + namespace p4est + { + struct InitFinalize + { + private: + struct Singleton + { + Singleton () + { + // ensure that the + // initialization + // code is run only + // once, even if we + // link with 1d, 2d, + // and 3d libraries + static bool initialized = false; + + if (initialized == false) + { + sc_init (MPI_COMM_WORLD, + 0, 0, 0, SC_LP_STATISTICS); + p4est_init (0, SC_LP_STATISTICS); + + initialized = true; + } + } + + ~Singleton () + { + // same here + static bool deinitialized = false; + + if (deinitialized == false) + { + // p4est has no + // p4est_finalize + // function + sc_finalize (); + + deinitialized = true; + } + } + }; + + public: + // do run the initialization code, at least the first time around we + // get to this function + static void do_initialize () + { + static Singleton singleton; + } + }; + } +} + + +namespace parallel +{ + namespace distributed + { + +/* ---------------------- class Triangulation ------------------------------ */ + + + template + Triangulation:: + Triangulation (MPI_Comm mpi_communicator, + const typename dealii::Triangulation::MeshSmoothing smooth_grid) + : + // do not check + // for distorted + // cells + dealii::Triangulation + (smooth_grid, + false), + mpi_communicator (Utilities::System:: + duplicate_communicator(mpi_communicator)), + my_subdomain (Utilities::System::get_this_mpi_process (this->mpi_communicator)), + triangulation_has_content (false), + connectivity (0), + parallel_forest (0), + refinement_in_progress (false), + attached_data_size(0), + n_attached_datas(0) + { + // initialize p4est. do this in a separate function since it has + // to happen only once, even if we have triangulation objects + // for several different space dimensions + internal::p4est::InitFinalize::do_initialize (); + + number_cache.n_locally_owned_active_cells + .resize (Utilities::System::get_n_mpi_processes (mpi_communicator)); + } + + + + template + Triangulation::~Triangulation () + { + clear (); + + Assert (triangulation_has_content == false, + ExcInternalError()); + Assert (connectivity == 0, ExcInternalError()); + Assert (parallel_forest == 0, ExcInternalError()); + Assert (refinement_in_progress == false, ExcInternalError()); + + // get rid of the unique + // communicator used here again + MPI_Comm_free (&mpi_communicator); + } + + + + + template + void + Triangulation:: + create_triangulation (const std::vector > &vertices, + const std::vector > &cells, + const SubCellData &subcelldata) + { + try + { + dealii::Triangulation:: + create_triangulation (vertices, cells, subcelldata); + } + catch (const typename dealii::Triangulation::DistortedCellList &) + { + // the underlying + // triangulation should not + // be checking for + // distorted cells + AssertThrow (false, ExcInternalError()); + } + + // note that now we have some content in + // the p4est objects and call the + // functions that do the actual work + // (which are dimension dependent, so + // separate) + triangulation_has_content = true; + + setup_coarse_cell_to_p4est_tree_permutation (); + + copy_new_triangulation_to_p4est (internal::int2type()); + + try + { + copy_local_forest_to_triangulation (); + } + catch (const typename Triangulation::DistortedCellList &) + { + // the underlying + // triangulation should not + // be checking for + // distorted cells + AssertThrow (false, ExcInternalError()); + } + + update_number_cache (); + } + + + + template + void + Triangulation::clear () + { + triangulation_has_content = false; + + if (parallel_forest != 0) + { + internal::p4est::functions::destroy (parallel_forest); + parallel_forest = 0; + } + + if (connectivity != 0) + { + internal::p4est::functions::connectivity_destroy (connectivity); + connectivity = 0; + } + + coarse_cell_to_p4est_tree_permutation.resize (0); + p4est_tree_to_coarse_cell_permutation.resize (0); + + dealii::Triangulation::clear (); + } + + + + template + void + Triangulation::setup_coarse_cell_to_p4est_tree_permutation () + { + SparsityPattern cell_connectivity; + GridTools::get_face_connectivity_of_cells (*this, cell_connectivity); + coarse_cell_to_p4est_tree_permutation.resize (this->n_cells(0)); + SparsityTools:: + reorder_Cuthill_McKee (cell_connectivity, + coarse_cell_to_p4est_tree_permutation); + + p4est_tree_to_coarse_cell_permutation + = Utilities::invert_permutation (coarse_cell_to_p4est_tree_permutation); + } + + + + template + void + Triangulation::write_mesh_vtk (const char *file_basename) const + { + Assert (parallel_forest != 0, + ExcMessage ("Can't produce output when no forest is created yet.")); + internal::p4est::functions:: + vtk_write_file (parallel_forest, 0, file_basename); + } + + + + template + unsigned int + Triangulation::get_checksum () const + { + Assert (parallel_forest != 0, + ExcMessage ("Can't produce a check sum when no forest is created yet.")); + return internal::p4est::functions::checksum (parallel_forest); + } + + + template + typename dealii::internal::p4est::types::tree * + Triangulation:: + init_tree(const int dealii_coarse_cell_index) const + { + const unsigned int tree_index + = coarse_cell_to_p4est_tree_permutation[dealii_coarse_cell_index]; + typename dealii::internal::p4est::types::tree *tree + = static_cast::tree*> + (sc_array_index (parallel_forest->trees, + tree_index)); + + return tree; + } + + + +#if deal_II_dimension == 2 + + template + void + Triangulation::copy_new_triangulation_to_p4est (internal::int2type<2>) + { + Assert (this->n_cells(0) > 0, ExcInternalError()); + Assert (this->n_levels() == 1, ExcInternalError()); + + // data structures that counts how many + // cells touch each vertex + // (vertex_touch_count), and which cells + // touch a given vertex (together with + // the local numbering of that vertex + // within the cells that touch it) + std::vector vertex_touch_count; + std::vector< + std::list< + std::pair::active_cell_iterator, + unsigned int> > > + vertex_to_cell; + get_vertex_to_cell_mappings (*this, + vertex_touch_count, + vertex_to_cell); + const internal::p4est::types<2>::locidx + num_vtt = std::accumulate (vertex_touch_count.begin(), + vertex_touch_count.end(), + 0); + + // now create a connectivity + // object with the right sizes + // for all arrays. set vertex + // information only in debug + // mode (saves a few bytes in + // optimized mode) + const bool set_vertex_info +#ifdef DEBUG + = true +#else + = false +#endif + ; + + connectivity + = internal::p4est::functions<2>:: + connectivity_new ((set_vertex_info == true ? this->n_vertices() : 0), + this->n_cells(0), + this->n_vertices(), + num_vtt); + + set_vertex_and_cell_info (*this, + vertex_touch_count, + vertex_to_cell, + coarse_cell_to_p4est_tree_permutation, + set_vertex_info, + connectivity); + + Assert (p4est_connectivity_is_valid (connectivity) == 1, + ExcInternalError()); + + // now create a forest out of the + // connectivity data structure + parallel_forest + = internal::p4est::functions<2>:: + new_forest (mpi_communicator, + connectivity, + /* minimum initial number of quadrants per tree */ 0, + /* minimum level of upfront refinement */ 0, + /* use uniform upfront refinement */ 1, + /* user_data_size = */ 0, + /* user_data_constructor = */ NULL, + /* user_pointer */ this); + } + +#endif + +#if deal_II_dimension == 3 + + template + void + Triangulation::copy_new_triangulation_to_p4est (internal::int2type<3>) + { + Assert (this->n_cells(0) > 0, ExcInternalError()); + Assert (this->n_levels() == 1, ExcInternalError()); + + // data structures that counts how many + // cells touch each vertex + // (vertex_touch_count), and which cells + // touch a given vertex (together with + // the local numbering of that vertex + // within the cells that touch it) + std::vector vertex_touch_count; + std::vector< + std::list< + std::pair::active_cell_iterator, + unsigned int> > > + vertex_to_cell; + get_vertex_to_cell_mappings (*this, + vertex_touch_count, + vertex_to_cell); + const internal::p4est::types<2>::locidx + num_vtt = std::accumulate (vertex_touch_count.begin(), + vertex_touch_count.end(), + 0); + + std::vector edge_touch_count; + std::vector< + std::list< + std::pair::active_cell_iterator, + unsigned int> > > + edge_to_cell; + get_edge_to_cell_mappings (*this, + edge_touch_count, + edge_to_cell); + const internal::p4est::types<2>::locidx + num_ett = std::accumulate (edge_touch_count.begin(), + edge_touch_count.end(), + 0); + // now create a connectivity object with + // the right sizes for all arrays + + const bool set_vertex_info +#ifdef DEBUG + = true +#else + = false +#endif + ; + + connectivity + = internal::p4est::functions<3>:: + connectivity_new ((set_vertex_info == true ? this->n_vertices() : 0), + this->n_cells(0), + this->n_active_lines(), + num_ett, + this->n_vertices(), + num_vtt); + + set_vertex_and_cell_info (*this, + vertex_touch_count, + vertex_to_cell, + coarse_cell_to_p4est_tree_permutation, + set_vertex_info, + connectivity); + + // next to tree-to-edge + // data. note that in p4est lines + // are ordered as follows + // *---3---* *---3---* + // /| | / /| + // 6 | 11 6 7 11 + // / 10 | / / | + // * | | *---2---* | + // | *---1---* | | * + // | / / | 9 / + // 8 4 5 8 | 5 + // |/ / | |/ + // *---0---* *---0---* + // whereas in deal.II they are like this: + // *---7---* *---7---* + // /| | / /| + // 4 | 11 4 5 11 + // / 10 | / / | + // * | | *---6---* | + // | *---3---* | | * + // | / / | 9 / + // 8 0 1 8 | 1 + // |/ / | |/ + // *---2---* *---2---* + + const unsigned int deal_to_p4est_line_index[12] + = { 4, 5, 0, 1, 6, 7, 2, 3, 8, 9, 10, 11 } ; + + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); ++cell) + { + const unsigned int + index = coarse_cell_to_p4est_tree_permutation[cell->index()]; + for (unsigned int e=0; e::lines_per_cell; ++e) + connectivity->tree_to_edge[index*GeometryInfo<3>::lines_per_cell+ + deal_to_p4est_line_index[e]] + = cell->line(e)->index(); + } + + // now also set edge-to-tree + // information + connectivity->ett_offset[0] = 0; + std::partial_sum (edge_touch_count.begin(), + edge_touch_count.end(), + &connectivity->ett_offset[1]); + + Assert (connectivity->ett_offset[this->n_active_lines()] == + num_ett, + ExcInternalError()); + + for (unsigned int v=0; vn_active_lines(); ++v) + { + Assert (edge_to_cell[v].size() == edge_touch_count[v], + ExcInternalError()); + + typename std::list::active_cell_iterator, + unsigned int> >::const_iterator + p = edge_to_cell[v].begin(); + for (unsigned int c=0; cedge_to_tree[connectivity->ett_offset[v]+c] + = coarse_cell_to_p4est_tree_permutation[p->first->index()]; + connectivity->edge_to_edge[connectivity->ett_offset[v]+c] + = deal_to_p4est_line_index[p->second]; + } + } + + Assert (p8est_connectivity_is_valid (connectivity) == 1, + ExcInternalError()); + + // now create a forest out of the + // connectivity data structure + parallel_forest + = internal::p4est::functions<3>:: + new_forest (mpi_communicator, + connectivity, + /* minimum initial number of quadrants per tree */ 0, + /* minimum level of upfront refinement */ 0, + /* use uniform upfront refinement */ 1, + /* user_data_size = */ 0, + /* user_data_constructor = */ NULL, + /* user_pointer */ this); + } + +#endif + + + template + void + Triangulation::copy_local_forest_to_triangulation () + { + // disable mesh smoothing for + // recreating the deal.II + // triangulation, otherwise we might + // not be able to reproduce the p4est + // mesh exactly. We restore the + // original smoothing at the end of + // this function. Note that the + // smoothing flag is used in the normal + // refinement process. + typename Triangulation::MeshSmoothing + save_smooth = this->smooth_grid; + this->smooth_grid = dealii::Triangulation::none; + bool mesh_changed = false; + + // query p4est for the ghost cells + typename internal::p4est::types::ghost * ghostlayer; + ghostlayer = internal::p4est::functions::ghost_new (parallel_forest, + (dim == 2 + ? + typename internal::p4est::types:: + balance_type(P4EST_BALANCE_CORNER) + : + typename internal::p4est::types:: + balance_type(P8EST_BALANCE_CORNER))); + + Assert (ghostlayer, ExcInternalError()); + + + //set all cells to artificial + for (typename Triangulation::cell_iterator + cell = this->begin(0); + cell != this->end(0); + ++cell) + cell->recursively_set_subdomain_id(types::artificial_subdomain_id); + + do + { + for (typename Triangulation::cell_iterator + cell = this->begin(0); + cell != this->end(0); + ++cell) + { + // if this processor + // stores no part of the + // forest that comes out + // of this coarse grid + // cell, then we need to + // delete all children of + // this cell (the coarse + // grid cell remains) + if (tree_exists_locally(parallel_forest, + coarse_cell_to_p4est_tree_permutation[cell->index()]) + == false) + { + delete_all_children (cell); + cell->set_subdomain_id (types::artificial_subdomain_id); + } + + else + { + + // this processor + // stores at least a + // part of the tree + // that comes out of + // this cell. + + + typename internal::p4est::types::quadrant p4est_coarse_cell; + typename internal::p4est::types::tree *tree = + init_tree(cell->index()); + + internal::p4est::init_coarse_quadrant(p4est_coarse_cell); + + match_tree_recursively (*tree, cell, + p4est_coarse_cell, + *parallel_forest, + my_subdomain); + } + } + + // check mesh for ghostcells, + // refine as neccessary. + // iterate over every ghostquadrant, + // find corresponding deal coarsecell + // and recurse. + typename internal::p4est::types::quadrant * quadr; + unsigned int ghost_owner=0; + + for (unsigned int g_idx=0;g_idxghosts.elem_count;++g_idx) + { + while (g_idx >= (unsigned int)ghostlayer->proc_offsets[ghost_owner+1]) + ++ghost_owner; + + quadr = static_cast::quadrant *> + ( sc_array_index(&ghostlayer->ghosts, g_idx) ); + + unsigned int coarse_cell_index = + p4est_tree_to_coarse_cell_permutation[quadr->p.piggy3.which_tree]; + + const typename Triangulation::cell_iterator + cell (this, 0U, coarse_cell_index); + + typename internal::p4est::types::quadrant p4est_coarse_cell; + internal::p4est::init_coarse_quadrant (p4est_coarse_cell); + + match_quadrant_recursively (p4est_coarse_cell, cell, *quadr, + *parallel_forest, ghost_owner); + } + + // fix all the flags to make + // sure we have a consistent + // mesh + this->prepare_coarsening_and_refinement (); + + // see if any flags are still set + mesh_changed = false; + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); + ++cell) + if (cell->refine_flag_set() || cell->coarsen_flag_set()) + { + mesh_changed = true; + break; + } + + // actually do the refinement + // but prevent the refinement + // hook below from taking + // over + const bool saved_refinement_in_progress = refinement_in_progress; + refinement_in_progress = true; + + try + { + this->execute_coarsening_and_refinement(); + } + catch (const typename Triangulation::DistortedCellList &) + { + // the underlying + // triangulation should not + // be checking for + // distorted cells + AssertThrow (false, ExcInternalError()); + } + + refinement_in_progress = saved_refinement_in_progress; + } + while (mesh_changed); + +#ifdef DEBUG + // check if correct number of ghosts is created + unsigned int num_ghosts = 0; + + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); + ++cell) + { + if (cell->subdomain_id() != my_subdomain + && + cell->subdomain_id() != types::artificial_subdomain_id) + ++num_ghosts; + } + + Assert( num_ghosts == ghostlayer->ghosts.elem_count, ExcInternalError()); +#endif + + + + // check that our local copy has + // exactly as many cells as the + // p4est original (at least if we + // are on only one processor); + // for parallel computations, we + // want to check that we have at + // least as many as p4est stores + // locally (in the future we + // should check that we have + // exactly as many non-artificial + // cells as + // parallel_forest->local_num_quadrants) + { + const unsigned int total_local_cells = this->n_active_cells(); + + if (Utilities::System::get_n_mpi_processes (mpi_communicator) == 1) + Assert (static_cast(parallel_forest->local_num_quadrants) == + total_local_cells, + ExcInternalError()) + else + Assert (static_cast(parallel_forest->local_num_quadrants) <= + total_local_cells, + ExcInternalError()); + + // count the number of owned, active + // cells and compare with p4est. + unsigned int n_owned = 0; + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); ++cell) + { + if (cell->subdomain_id() == my_subdomain) + ++n_owned; + } + + Assert(static_cast(parallel_forest->local_num_quadrants) == + n_owned, ExcInternalError()); + + } + + internal::p4est::functions::ghost_destroy (ghostlayer); + + this->smooth_grid = save_smooth; + } + + + + template + void + Triangulation::execute_coarsening_and_refinement () + { + // first make sure that recursive + // calls are handled correctly + if (refinement_in_progress == true) + { + dealii::Triangulation::execute_coarsening_and_refinement (); + return; + } + + Timer t(MPI_COMM_WORLD, true); + t.start(); + + + + // now do the work we're + // supposed to do when we are + // in charge + refinement_in_progress = true; + this->prepare_coarsening_and_refinement (); + + // make sure all flags are + // cleared on cells we don't + // own, since nothing good can + // come of that if they are + // still around + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); ++cell) + if (cell->is_ghost() || cell->is_artificial()) + { + cell->clear_refine_flag (); + cell->clear_coarsen_flag (); + } + + + + // count how many cells will be refined + // and coarsened, and allocate that much + // memory + RefineAndCoarsenList + refine_and_coarsen_list (*this, + p4est_tree_to_coarse_cell_permutation, + my_subdomain, + *parallel_forest); + + // copy refine and coarsen flags into + // p4est and execute the refinement and + // coarsening. this uses the + // refine_and_coarsen_list just built, + // which is communicated to the callback + // functions through the + // user_pointer + Assert (parallel_forest->user_pointer == this, + ExcInternalError()); + parallel_forest->user_pointer = &refine_and_coarsen_list; + + internal::p4est::functions:: + refine (parallel_forest, /* refine_recursive */ false, + &RefineAndCoarsenList::refine_callback, + /*init_callback=*/NULL); + internal::p4est::functions:: + coarsen (parallel_forest, /* coarsen_recursive */ false, + &RefineAndCoarsenList::coarsen_callback, + /*init_callback=*/NULL); + + t.stop(); +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (my_subdomain==0) + { + deallog << "_p4est::ref&coarsen "; + t.print_data(deallog); + } +#endif + + t.reset(); + t.start(); + + + // make sure all cells in the lists have + // been consumed + Assert (refine_and_coarsen_list.pointers_are_at_end(), + ExcInternalError()); + + // reset the pointer + parallel_forest->user_pointer = this; + + // enforce 2:1 hanging node condition + internal::p4est::functions:: + balance (parallel_forest, + /* face and corner balance */ + (dim == 2 + ? + typename internal::p4est::types:: + balance_type(P4EST_BALANCE_FULL) + : + typename internal::p4est::types:: + balance_type(P8EST_BALANCE_FULL)), + /*init_callback=*/NULL); + + + t.stop(); +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (my_subdomain==0) + { + deallog << "_p4est::balance "; + t.print_data(deallog); + } +#endif + t.reset(); + t.start(); + + + // before repartitioning the mesh let + // others attach mesh related info + // (such as SolutionTransfer data) to + // the p4est + attach_mesh_data(); + + + t.stop(); +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (my_subdomain==0) + { + deallog << "_attach_mesh_data "; + t.print_data(deallog); + } +#endif + t.reset(); + t.start(); + + // partition the new mesh between + // all processors + internal::p4est::functions:: + partition (parallel_forest, + /* prepare coarsening */ 1, + /* weight_callback */ NULL); + + t.stop(); +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (my_subdomain==0) + { + deallog << "_p4est::partition "; + t.print_data(deallog); + } +#endif + t.reset(); + t.start(); + + // finally copy back from local + // part of tree to deal.II + // triangulation. before doing + // so, make sure there are no + // refine or coarsen flags + // pending + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); ++cell) + { + cell->clear_refine_flag(); + cell->clear_coarsen_flag(); + } + + try + { + copy_local_forest_to_triangulation (); + } + catch (const typename Triangulation::DistortedCellList &) + { + // the underlying + // triangulation should not + // be checking for + // distorted cells + AssertThrow (false, ExcInternalError()); + } + + + refinement_in_progress = false; + + update_number_cache (); + + t.stop(); +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (my_subdomain==0) + { + deallog << "_copy_to_deal "; + t.print_data(deallog); + } +#endif + t.reset(); + } + + + + template + void + Triangulation::update_number_cache () + { + Assert (number_cache.n_locally_owned_active_cells.size() + == + Utilities::System::get_n_mpi_processes (mpi_communicator), + ExcInternalError()); + + std::fill (number_cache.n_locally_owned_active_cells.begin(), + number_cache.n_locally_owned_active_cells.end(), + 0); + + for (typename Triangulation::active_cell_iterator + cell = this->begin_active(); + cell != this->end(); ++cell) + if (cell->subdomain_id() == my_subdomain) + ++number_cache.n_locally_owned_active_cells[my_subdomain]; + + unsigned int send_value + = number_cache.n_locally_owned_active_cells[my_subdomain]; + MPI_Allgather (&send_value, + 1, + MPI_UNSIGNED, + &number_cache.n_locally_owned_active_cells[0], + 1, + MPI_UNSIGNED, + mpi_communicator); + + number_cache.n_global_active_cells + = std::accumulate (number_cache.n_locally_owned_active_cells.begin(), + number_cache.n_locally_owned_active_cells.end(), + 0); + } + + + + template + types::subdomain_id_t + Triangulation::locally_owned_subdomain () const + { + return my_subdomain; + } + + + + template + unsigned int + Triangulation::n_locally_owned_active_cells () const + { + return number_cache.n_locally_owned_active_cells[my_subdomain]; + } + + + + template + unsigned int + Triangulation::n_global_active_cells () const + { + return number_cache.n_global_active_cells; + } + + + + template + const std::vector & + Triangulation::n_locally_owned_active_cells_per_processor () const + { + return number_cache.n_locally_owned_active_cells; + } + + + + template + unsigned int + Triangulation:: + register_data_attach (const size_t size, + const std_cxx1x::function & pack_callback) + { + Assert(size>0, ExcMessage("register_data_attach(), size==0")); + Assert(attached_data_pack_callbacks.size()==n_attached_datas, + ExcMessage("register_data_attach(), not all data has been unpacked last time?")); + + unsigned int offset = attached_data_size+sizeof(CellStatus); + ++n_attached_datas; + attached_data_size+=size; + attached_data_pack_callbacks.push_back( + std::pair (offset, pack_callback) + ); + return offset; + } + + + + template + void + Triangulation:: + notify_ready_to_unpack (const unsigned int offset, + const std_cxx1x::function & unpack_callback) + { + Assert(offset < attached_data_size, ExcMessage("invalid offset in notify_ready_to_unpack()")); + Assert(n_attached_datas>0, ExcMessage("notify_ready_to_unpack() called too often")); + + // Recurse over p4est and hand the caller the data back + for (typename Triangulation::cell_iterator + cell = this->begin(0); + cell != this->end(0); + ++cell) + { + //skip coarse cells, that are not ours + if (tree_exists_locally(parallel_forest, + coarse_cell_to_p4est_tree_permutation[cell->index()]) + == false) + continue; + + typename internal::p4est::types::quadrant p4est_coarse_cell; + typename internal::p4est::types::tree *tree = + init_tree(cell->index()); + + internal::p4est::init_coarse_quadrant (p4est_coarse_cell); + + // parent_cell is not correct here, + // but is only used in a refined + // cell + post_mesh_data_recursively(*tree, + cell, + cell, + p4est_coarse_cell, + offset, + unpack_callback); + } + + --n_attached_datas; + + if (!n_attached_datas) + { + // everybody got his data, time for cleanup! + attached_data_size=0; + attached_data_pack_callbacks.clear(); + + // and release the data + void * userptr = parallel_forest->user_pointer; + internal::p4est::functions::reset_data (parallel_forest, 0, NULL, NULL); + parallel_forest->user_pointer = userptr; + } + } + + + + template + MPI_Comm + Triangulation::get_communicator () const + { + return mpi_communicator; + } + + + + template + unsigned int + Triangulation::memory_consumption () const + { + unsigned int mem= + this->dealii::Triangulation::memory_consumption() + + MemoryConsumption::memory_consumption(mpi_communicator) + + MemoryConsumption::memory_consumption(my_subdomain) + + MemoryConsumption::memory_consumption(triangulation_has_content) + + MemoryConsumption::memory_consumption(number_cache.n_locally_owned_active_cells) + + MemoryConsumption::memory_consumption(number_cache.n_global_active_cells) + + MemoryConsumption::memory_consumption(connectivity) + + MemoryConsumption::memory_consumption(parallel_forest) + + MemoryConsumption::memory_consumption(refinement_in_progress) + + MemoryConsumption::memory_consumption(attached_data_size) + + MemoryConsumption::memory_consumption(n_attached_datas) +// + MemoryConsumption::memory_consumption(attached_data_pack_callbacks) //TODO[TH]: how? + + MemoryConsumption::memory_consumption(coarse_cell_to_p4est_tree_permutation) + + MemoryConsumption::memory_consumption(p4est_tree_to_coarse_cell_permutation) + + memory_consumption_p4est() + ; + + return mem; + } + + + + template + unsigned int + Triangulation::memory_consumption_p4est () const + { + return internal::p4est::functions::forest_memory_used(parallel_forest) + + internal::p4est::functions::connectivity_memory_used(connectivity); + } + + + + template + void + Triangulation:: + copy_triangulation (const dealii::Triangulation &) + { + Assert (false, ExcNotImplemented()); + } + + + template + void + Triangulation:: + attach_mesh_data() + { + // determine size of memory in bytes to + // attach to each cell. This needs to + // be constant because of p4est. + if (attached_data_size==0) + { + Assert(n_attached_datas==0, ExcInternalError()); + + //nothing to do + return; + } + + // realloc user_data in p4est + void * userptr = parallel_forest->user_pointer; + internal::p4est::functions::reset_data (parallel_forest, + attached_data_size+sizeof(CellStatus), + NULL, NULL); + parallel_forest->user_pointer = userptr; + + + // Recurse over p4est and Triangulation + // to find refined/coarsened/kept + // cells. Then query and attach the data. + for (typename Triangulation::cell_iterator + cell = this->begin(0); + cell != this->end(0); + ++cell) + { + //skip coarse cells, that are not ours + if (tree_exists_locally(parallel_forest, + coarse_cell_to_p4est_tree_permutation[cell->index()]) + == false) + continue; + + typename internal::p4est::types::quadrant p4est_coarse_cell; + typename internal::p4est::types::tree *tree = + init_tree(cell->index()); + + internal::p4est::init_coarse_quadrant(p4est_coarse_cell); + + attach_mesh_data_recursively(*tree, + cell, + p4est_coarse_cell, + attached_data_pack_callbacks); + } + + + + + } + + + + +#if deal_II_dimension == 1 + template + Triangulation<1,spacedim>::Triangulation (MPI_Comm) + { + Assert (false, ExcNotImplemented()); + } + + + template + Triangulation<1,spacedim>::~Triangulation () + { + Assert (false, ExcNotImplemented()); + } + + + + template + types::subdomain_id_t + Triangulation<1,spacedim>::locally_owned_subdomain () const + { + Assert (false, ExcNotImplemented()); + return 0; + } + + + template + MPI_Comm + Triangulation<1,spacedim>::get_communicator () const + { + return MPI_COMM_WORLD; + } +#endif + + } +} + + +#else // DEAL_II_USE_P4EST + +namespace parallel +{ + namespace distributed + { + template + Triangulation::Triangulation () + { + Assert (false, ExcNotImplemented()); + } + + + template + Triangulation::~Triangulation () + { + Assert (false, ExcNotImplemented()); + } + + + + template + types::subdomain_id_t + Triangulation::locally_owned_subdomain () const + { + Assert (false, ExcNotImplemented()); + return 0; + } + } +} + + +#endif // DEAL_II_USE_P4EST + + + +// explicit instantiations +namespace internal +{ + namespace p4est + { +# if deal_II_dimension > 1 + template + void + init_quadrant_children + (const types::quadrant & p4est_cell, + types::quadrant (&p4est_children)[GeometryInfo::max_children_per_cell]); + + template + void + init_coarse_quadrant + (types::quadrant & quad); + + template + bool + quadrant_is_equal + (const types::quadrant & q1, + const types::quadrant & q2); + + template + bool + quadrant_is_ancestor + (const types::quadrant & q1, + const types::quadrant & q2); +#endif + } +} +namespace parallel +{ + namespace distributed + { + template class Triangulation; +# if deal_II_dimension < 3 + template class Triangulation; +# endif + } +} + + +DEAL_II_NAMESPACE_CLOSE + + diff --git a/deal.II/deal.II/source/dofs/dof_accessor.cc b/deal.II/deal.II/source/dofs/dof_accessor.cc index 779665d4c0..2cff208515 100644 --- a/deal.II/deal.II/source/dofs/dof_accessor.cc +++ b/deal.II/deal.II/source/dofs/dof_accessor.cc @@ -45,15 +45,33 @@ DoFCellAccessor::update_cell_dof_indices_cache () const { Assert (static_cast(this->present_level) < this->dof_handler->levels.size(), ExcMessage ("DoFHandler not initialized")); - + + Assert (this->dof_handler != 0, typename BaseClass::ExcInvalidObject()); + Assert (&this->get_fe() != 0, typename BaseClass::ExcInvalidObject()); + + internal::DoFCellAccessor::Implementation:: + update_cell_dof_indices_cache (*this); +} + + + +template +void +DoFCellAccessor::set_dof_indices (const std::vector &local_dof_indices) +{ + Assert (static_cast(this->present_level) < this->dof_handler->levels.size(), + ExcMessage ("DoFHandler not initialized")); + Assert (this->dof_handler != 0, typename BaseClass::ExcInvalidObject()); Assert (&this->get_fe() != 0, typename BaseClass::ExcInvalidObject()); - internal::DoFCellAccessor::Implementation::update_cell_dof_indices_cache (*this); + internal::DoFCellAccessor::Implementation:: + set_dof_indices (*this, local_dof_indices); } + template typename internal::DoFHandler::Iterators::cell_iterator DoFCellAccessor::neighbor_child_on_subface (const unsigned int face, @@ -79,7 +97,7 @@ get_interpolated_dof_values (const InputVector &values, { const FiniteElement &fe = this->get_fe(); const unsigned int dofs_per_cell = fe.dofs_per_cell; - + Assert (this->dof_handler != 0, typename BaseClass::ExcInvalidObject()); Assert (&fe != 0, @@ -98,7 +116,7 @@ get_interpolated_dof_values (const InputVector &values, { Vector tmp1(dofs_per_cell); Vector tmp2(dofs_per_cell); - + interpolated_values = 0; // later on we will have to @@ -161,7 +179,7 @@ get_interpolated_dof_values (const InputVector &values, std::vector restriction_is_additive (dofs_per_cell); for (unsigned int i=0; in_children(); ++child) { // get the values from the present @@ -176,7 +194,7 @@ get_interpolated_dof_values (const InputVector &values, // and add up or set them // in the output vector for (unsigned int i=0; i &local_values, // otherwise distribute them to the children { Vector tmp(dofs_per_cell); - + for (unsigned int child=0; childn_children(); ++child) { // prolong the given data // to the present cell this->get_fe().get_prolongation_matrix(child, this->refinement_case()) .vmult (tmp, local_values); - + this->child(child)->set_dof_values_by_interpolation (tmp, values); } } @@ -324,11 +342,11 @@ template class TriaActiveIterator > >; #if deal_II_dimension != 3 template class DoFCellAccessor >; -template class +template class TriaRawIterator > >; -template class +template class TriaIterator > >; -template class +template class TriaActiveIterator > >; #endif @@ -351,11 +369,11 @@ template class TriaActiveIterator > >; #if deal_II_dimension != 3 template class DoFCellAccessor >; -template class +template class TriaRawIterator > >; -template class +template class TriaIterator > >; -template class +template class TriaActiveIterator > >; #endif diff --git a/deal.II/deal.II/source/dofs/dof_handler.cc b/deal.II/deal.II/source/dofs/dof_handler.cc index 3a0757a1cc..729bee2f4c 100644 --- a/deal.II/deal.II/source/dofs/dof_handler.cc +++ b/deal.II/deal.II/source/dofs/dof_handler.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -71,343 +72,13 @@ namespace internal // namespace internal::DoFHandler using dealii::DoFHandler; + /** * A class with the same purpose as the similarly named class of the * Triangulation class. See there for more information. */ struct Implementation { - /** - * Distribute dofs on the given cell, - * with new dofs starting with index - * @p next_free_dof. Return the next - * unused index number. The finite - * element used is the one given to - * @p distribute_dofs, which is copied - * to @p selected_fe. - * - * This function is excluded from the - * @p distribute_dofs function since - * it can not be implemented dimension - * independent. - */ - template - static - unsigned int - distribute_dofs_on_cell (const DoFHandler<1,spacedim> &dof_handler, - typename DoFHandler<1,spacedim>::active_cell_iterator &cell, - unsigned int next_free_dof) - { - - // distribute dofs of vertices - for (unsigned int v=0; v::vertices_per_cell; ++v) - { - typename DoFHandler<1,spacedim>::cell_iterator - neighbor = cell->neighbor(v); - - if (neighbor.state() == IteratorState::valid) - { - // find true neighbor; may be its - // a child of @p{neighbor} - while (neighbor->has_children()) - neighbor = neighbor->child(v==0 ? 1 : 0); - - // has neighbor already been processed? - if (neighbor->user_flag_set()) - // copy dofs - { - if (v==0) - for (unsigned int d=0; - ddofs_per_vertex; ++d) - cell->set_vertex_dof_index (0, d, - neighbor->vertex_dof_index (1, d)); - else - for (unsigned int d=0; - ddofs_per_vertex; ++d) - cell->set_vertex_dof_index (1, d, - neighbor->vertex_dof_index (0, d)); - - // next neighbor - continue; - } - } - - // otherwise: create dofs newly - for (unsigned int d=0; - ddofs_per_vertex; ++d) - cell->set_vertex_dof_index (v, d, next_free_dof++); - } - - // dofs of line - for (unsigned int d=0; - ddofs_per_line; ++d) - cell->set_dof_index (d, next_free_dof++); - - // note that this cell has been - // processed - cell->set_user_flag (); - - return next_free_dof; - } - - - - template - static - unsigned int - distribute_dofs_on_cell (const DoFHandler<2,spacedim> &dof_handler, - typename DoFHandler<2,spacedim>::active_cell_iterator &cell, - unsigned int next_free_dof) - { - if (dof_handler.selected_fe->dofs_per_vertex > 0) - // number dofs on vertices - for (unsigned int vertex=0; vertex::vertices_per_cell; ++vertex) - // check whether dofs for this - // vertex have been distributed - // (only check the first dof) - if (cell->vertex_dof_index(vertex, 0) == DoFHandler<2,spacedim>::invalid_dof_index) - for (unsigned int d=0; ddofs_per_vertex; ++d) - cell->set_vertex_dof_index (vertex, d, next_free_dof++); - - // for the four sides - if (dof_handler.selected_fe->dofs_per_line > 0) - for (unsigned int side=0; side::faces_per_cell; ++side) - { - typename DoFHandler<2,spacedim>::line_iterator - line = cell->line(side); - - // distribute dofs if necessary: - // check whether line dof is already - // numbered (check only first dof) - if (line->dof_index(0) == DoFHandler<2,spacedim>::invalid_dof_index) - // if not: distribute dofs - for (unsigned int d=0; ddofs_per_line; ++d) - line->set_dof_index (d, next_free_dof++); - } - - - // dofs of quad - if (dof_handler.selected_fe->dofs_per_quad > 0) - for (unsigned int d=0; ddofs_per_quad; ++d) - cell->set_dof_index (d, next_free_dof++); - - - // note that this cell has been processed - cell->set_user_flag (); - - return next_free_dof; - } - - - template - static - unsigned int - distribute_dofs_on_cell (const DoFHandler<3,spacedim> &dof_handler, - typename DoFHandler<3,spacedim>::active_cell_iterator &cell, - unsigned int next_free_dof) - { - if (dof_handler.selected_fe->dofs_per_vertex > 0) - // number dofs on vertices - for (unsigned int vertex=0; vertex::vertices_per_cell; ++vertex) - // check whether dofs for this - // vertex have been distributed - // (only check the first dof) - if (cell->vertex_dof_index(vertex, 0) == DoFHandler<3,spacedim>::invalid_dof_index) - for (unsigned int d=0; ddofs_per_vertex; ++d) - cell->set_vertex_dof_index (vertex, d, next_free_dof++); - - // for the lines - if (dof_handler.selected_fe->dofs_per_line > 0) - for (unsigned int l=0; l::lines_per_cell; ++l) - { - typename DoFHandler<3,spacedim>::line_iterator - line = cell->line(l); - - // distribute dofs if necessary: - // check whether line dof is already - // numbered (check only first dof) - if (line->dof_index(0) == DoFHandler<3,spacedim>::invalid_dof_index) - // if not: distribute dofs - for (unsigned int d=0; ddofs_per_line; ++d) - line->set_dof_index (d, next_free_dof++); - } - - // for the quads - if (dof_handler.selected_fe->dofs_per_quad > 0) - for (unsigned int q=0; q::quads_per_cell; ++q) - { - typename DoFHandler<3,spacedim>::quad_iterator - quad = cell->quad(q); - - // distribute dofs if necessary: - // check whether quad dof is already - // numbered (check only first dof) - if (quad->dof_index(0) == DoFHandler<3,spacedim>::invalid_dof_index) - // if not: distribute dofs - for (unsigned int d=0; ddofs_per_quad; ++d) - quad->set_dof_index (d, next_free_dof++); - } - - - // dofs of hex - if (dof_handler.selected_fe->dofs_per_hex > 0) - for (unsigned int d=0; ddofs_per_hex; ++d) - cell->set_dof_index (d, next_free_dof++); - - - // note that this cell has been - // processed - cell->set_user_flag (); - - return next_free_dof; - } - - - /** - * Implementation of the general template - * of same name. - */ - template - static - void renumber_dofs (const std::vector &new_numbers, - DoFHandler<1,spacedim> &dof_handler) - { - // note that we can not use cell - // iterators in this function since - // then we would renumber the dofs on - // the interface of two cells more - // than once. Anyway, this way it's - // not only more correct but also - // faster; note, however, that dof - // numbers may be invalid_dof_index, - // namely when the appropriate - // vertex/line/etc is unused - for (std::vector::iterator - i=dof_handler.vertex_dofs.begin(); - i!=dof_handler.vertex_dofs.end(); ++i) - if (*i != DoFHandler<1,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - else - // if index is - // invalid_dof_index: - // check if this one - // really is unused - Assert (dof_handler.tria - ->vertex_used((i-dof_handler.vertex_dofs.begin()) / - dof_handler.selected_fe->dofs_per_vertex) - == false, - ExcInternalError ()); - - for (unsigned int level=0; level::iterator - i=dof_handler.levels[level]->lines.dofs.begin(); - i!=dof_handler.levels[level]->lines.dofs.end(); ++i) - if (*i != DoFHandler<1,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - } - - - - template - static - void renumber_dofs (const std::vector &new_numbers, - DoFHandler<2,spacedim> &dof_handler) - { - // note that we can not use cell - // iterators in this function since - // then we would renumber the dofs on - // the interface of two cells more - // than once. Anyway, this way it's - // not only more correct but also - // faster; note, however, that dof - // numbers may be invalid_dof_index, - // namely when the appropriate - // vertex/line/etc is unused - for (std::vector::iterator - i=dof_handler.vertex_dofs.begin(); - i!=dof_handler.vertex_dofs.end(); ++i) - if (*i != DoFHandler<2,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - else - // if index is invalid_dof_index: - // check if this one really is - // unused - Assert (dof_handler.tria - ->vertex_used((i-dof_handler.vertex_dofs.begin()) / - dof_handler.selected_fe->dofs_per_vertex) - == false, - ExcInternalError ()); - - for (std::vector::iterator - i=dof_handler.faces->lines.dofs.begin(); - i!=dof_handler.faces->lines.dofs.end(); ++i) - if (*i != DoFHandler<2,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - - for (unsigned int level=0; level::iterator - i=dof_handler.levels[level]->quads.dofs.begin(); - i!=dof_handler.levels[level]->quads.dofs.end(); ++i) - if (*i != DoFHandler<2,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - } - } - - - template - static - void renumber_dofs (const std::vector &new_numbers, - DoFHandler<3,spacedim> &dof_handler) - { - // note that we can not use cell - // iterators in this function since - // then we would renumber the dofs on - // the interface of two cells more - // than once. Anyway, this way it's - // not only more correct but also - // faster; note, however, that dof - // numbers may be invalid_dof_index, - // namely when the appropriate - // vertex/line/etc is unused - for (std::vector::iterator - i=dof_handler.vertex_dofs.begin(); - i!=dof_handler.vertex_dofs.end(); ++i) - if (*i != DoFHandler<3,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - else - // if index is invalid_dof_index: - // check if this one really is - // unused - Assert (dof_handler.tria - ->vertex_used((i-dof_handler.vertex_dofs.begin()) / - dof_handler.selected_fe->dofs_per_vertex) - == false, - ExcInternalError ()); - - for (std::vector::iterator - i=dof_handler.faces->lines.dofs.begin(); - i!=dof_handler.faces->lines.dofs.end(); ++i) - if (*i != DoFHandler<3,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - for (std::vector::iterator - i=dof_handler.faces->quads.dofs.begin(); - i!=dof_handler.faces->quads.dofs.end(); ++i) - if (*i != DoFHandler<3,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - - for (unsigned int level=0; level::iterator - i=dof_handler.levels[level]->hexes.dofs.begin(); - i!=dof_handler.levels[level]->hexes.dofs.end(); ++i) - if (*i != DoFHandler<3,spacedim>::invalid_dof_index) - *i = new_numbers[*i]; - } - } - - - /** * Implement the function of same name in * the mother class. @@ -692,9 +363,18 @@ DoFHandler::DoFHandler (const Triangulation &tria) : tria(&tria, typeid(*this).name()), selected_fe(0, typeid(*this).name()), - faces(NULL), - used_dofs (0) -{} + faces(NULL) +{ + // decide whether we need a + // sequential or a parallel + // distributed policy + if (dynamic_cast*> + (&tria) + == 0) + policy.reset (new internal::DoFHandler::Policy::Sequential()); + else + policy.reset (new internal::DoFHandler::Policy::ParallelDistributed()); +} template @@ -702,8 +382,7 @@ DoFHandler::DoFHandler () : tria(0, typeid(*this).name()), selected_fe(0, typeid(*this).name()), - faces(NULL), - used_dofs (0) + faces(NULL) {} @@ -722,8 +401,19 @@ DoFHandler::initialize( const FiniteElement& fe) { tria = &t; - faces = NULL; - used_dofs = 0; + faces = 0; + number_cache.n_global_dofs = 0; + + // decide whether we need a + // sequential or a parallel + // distributed policy + if (dynamic_cast*> + (&t) + == 0) + policy.reset (new internal::DoFHandler::Policy::Sequential()); + else + policy.reset (new internal::DoFHandler::Policy::ParallelDistributed()); + distribute_dofs(fe); } @@ -1954,7 +1644,7 @@ DoFHandler::memory_consumption () const MemoryConsumption::memory_consumption (levels) + MemoryConsumption::memory_consumption (*faces) + MemoryConsumption::memory_consumption (faces) + - MemoryConsumption::memory_consumption (used_dofs) + + sizeof (number_cache) + MemoryConsumption::memory_consumption (vertex_dofs)); for (unsigned int i=0; i::memory_consumption () const template void DoFHandler:: - distribute_dofs (const FiniteElement &ff, - const unsigned int offset) +distribute_dofs (const FiniteElement &ff, + const unsigned int offset) { - Assert (tria->n_levels() > 0, ExcInvalidTriangulation()); - selected_fe = &ff; - // delete all levels and set them up - // newly, since vectors are - // troublesome if you want to change - // their size + // delete all levels and set them + // up newly. note that we still + // have to allocate space for all + // degrees of freedom on this mesh + // (including ghost and cells that + // are entirely stored on different + // processors), though we may not + // assign numbers to some of them + // (i.e. they will remain at + // invalid_dof_index). We need to + // allocate the space because we + // will want to be able to query + // the dof_indices on each cell, + // and simply be told that we don't + // know them on some cell (i.e. get + // back invalid_dof_index) clear_space (); internal::DoFHandler::Implementation::reserve_space (*this); - // Clear user flags because we will - // need them. But first we save - // them and make sure that we - // restore them later such that at - // the end of this function the - // Triangulation will be in the - // same state as it was at the - // beginning of this function. - std::vector user_flags; - tria->save_user_flags(user_flags); - const_cast &>(*tria).clear_user_flags (); - - unsigned int next_free_dof = offset; - active_cell_iterator cell = begin_active(), - endc = end(); - - for (; cell != endc; ++cell) - next_free_dof - = internal::DoFHandler::Implementation:: - distribute_dofs_on_cell (*this, cell, next_free_dof); + // hand things off to the policy + number_cache = policy->distribute_dofs (offset, + *this); - used_dofs = next_free_dof; - - // update the cache used - // for cell dof indices - for (cell_iterator cell = begin(); cell != end(); ++cell) - cell->update_cell_dof_indices_cache (); - - // finally restore the user flags - const_cast &>(*tria).load_user_flags(user_flags); - - block_info_object.initialize(*this); + // initialize the block info object + // only if this is a sequential + // triangulation. it doesn't work + // correctly yet if it is parallel + if (dynamic_cast*>(&*tria) == 0) + block_info_object.initialize(*this); } + template void DoFHandler::initialize_local_block_info () { @@ -2022,6 +1701,7 @@ void DoFHandler::initialize_local_block_info () } + template void DoFHandler::clear () { @@ -2039,12 +1719,20 @@ void DoFHandler:: renumber_dofs (const std::vector &new_numbers) { - Assert (new_numbers.size() == n_dofs(), ExcRenumberingIncomplete()); + Assert (new_numbers.size() == n_locally_owned_dofs(), + ExcRenumberingIncomplete()); #ifdef DEBUG // assert that the new indices are - // consecutively numbered - if (true) + // consecutively numbered if we are + // working on a single + // processor. this doesn't need to + // hold in the case of a parallel + // mesh since we map the interval + // [0...n_dofs()) into itself but + // only globally, not on each + // processor + if (n_locally_owned_dofs() == n_dofs()) { std::vector tmp(new_numbers); std::sort (tmp.begin(), tmp.end()); @@ -2053,16 +1741,13 @@ renumber_dofs (const std::vector &new_numbers) for (; p!=tmp.end(); ++p, ++i) Assert (*p == i, ExcNewNumbersNotConsecutive(i)); } + else + for (unsigned int i=0; iupdate_cell_dof_indices_cache (); - + number_cache = policy->renumber_dofs (new_numbers, *this); } diff --git a/deal.II/deal.II/source/dofs/dof_handler_policy.cc b/deal.II/deal.II/source/dofs/dof_handler_policy.cc new file mode 100644 index 0000000000..385eaa564b --- /dev/null +++ b/deal.II/deal.II/source/dofs/dof_handler_policy.cc @@ -0,0 +1,1590 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +DEAL_II_NAMESPACE_OPEN + + +namespace internal +{ + namespace DoFHandler + { + namespace Policy + { + // use class dealii::DoFHandler instead + // of namespace internal::DoFHandler in + // the following + using dealii::DoFHandler; + + struct Implementation + { + +/* -------------- distribute_dofs functionality ------------- */ + + /** + * Distribute dofs on the given cell, + * with new dofs starting with index + * @p next_free_dof. Return the next + * unused index number. + * + * This function is excluded from the + * @p distribute_dofs function since + * it can not be implemented dimension + * independent. + */ + template + static + unsigned int + distribute_dofs_on_cell (const DoFHandler<1,spacedim> &dof_handler, + const typename DoFHandler<1,spacedim>::active_cell_iterator &cell, + unsigned int next_free_dof) + { + + // distribute dofs of vertices + for (unsigned int v=0; v::vertices_per_cell; ++v) + { + typename DoFHandler<1,spacedim>::cell_iterator + neighbor = cell->neighbor(v); + + if (neighbor.state() == IteratorState::valid) + { + // find true neighbor; may be its + // a child of @p{neighbor} + while (neighbor->has_children()) + neighbor = neighbor->child(v==0 ? 1 : 0); + + // has neighbor already been processed? + if (neighbor->user_flag_set()) + // copy dofs + { + if (v==0) + for (unsigned int d=0; + dset_vertex_dof_index (0, d, + neighbor->vertex_dof_index (1, d)); + else + for (unsigned int d=0; + dset_vertex_dof_index (1, d, + neighbor->vertex_dof_index (0, d)); + + // next neighbor + continue; + } + } + + // otherwise: create dofs newly + for (unsigned int d=0; + dset_vertex_dof_index (v, d, next_free_dof++); + } + + // dofs of line + for (unsigned int d=0; + dset_dof_index (d, next_free_dof++); + + // note that this cell has been + // processed + cell->set_user_flag (); + + return next_free_dof; + } + + + + template + static + unsigned int + distribute_dofs_on_cell (const DoFHandler<2,spacedim> &dof_handler, + const typename DoFHandler<2,spacedim>::active_cell_iterator &cell, + unsigned int next_free_dof) + { + if (dof_handler.get_fe().dofs_per_vertex > 0) + // number dofs on vertices + for (unsigned int vertex=0; vertex::vertices_per_cell; ++vertex) + // check whether dofs for this + // vertex have been distributed + // (only check the first dof) + if (cell->vertex_dof_index(vertex, 0) == DoFHandler<2,spacedim>::invalid_dof_index) + for (unsigned int d=0; dset_vertex_dof_index (vertex, d, next_free_dof++); + + // for the four sides + if (dof_handler.get_fe().dofs_per_line > 0) + for (unsigned int side=0; side::faces_per_cell; ++side) + { + const typename DoFHandler<2,spacedim>::line_iterator + line = cell->line(side); + + // distribute dofs if necessary: + // check whether line dof is already + // numbered (check only first dof) + if (line->dof_index(0) == DoFHandler<2,spacedim>::invalid_dof_index) + // if not: distribute dofs + for (unsigned int d=0; dset_dof_index (d, next_free_dof++); + } + + + // dofs of quad + if (dof_handler.get_fe().dofs_per_quad > 0) + for (unsigned int d=0; dset_dof_index (d, next_free_dof++); + + + // note that this cell has been processed + cell->set_user_flag (); + + return next_free_dof; + } + + + template + static + unsigned int + distribute_dofs_on_cell (const DoFHandler<3,spacedim> &dof_handler, + const typename DoFHandler<3,spacedim>::active_cell_iterator &cell, + unsigned int next_free_dof) + { + if (dof_handler.get_fe().dofs_per_vertex > 0) + // number dofs on vertices + for (unsigned int vertex=0; vertex::vertices_per_cell; ++vertex) + // check whether dofs for this + // vertex have been distributed + // (only check the first dof) + if (cell->vertex_dof_index(vertex, 0) == DoFHandler<3,spacedim>::invalid_dof_index) + for (unsigned int d=0; dset_vertex_dof_index (vertex, d, next_free_dof++); + + // for the lines + if (dof_handler.get_fe().dofs_per_line > 0) + for (unsigned int l=0; l::lines_per_cell; ++l) + { + const typename DoFHandler<3,spacedim>::line_iterator + line = cell->line(l); + + // distribute dofs if necessary: + // check whether line dof is already + // numbered (check only first dof) + if (line->dof_index(0) == DoFHandler<3,spacedim>::invalid_dof_index) + // if not: distribute dofs + for (unsigned int d=0; dset_dof_index (d, next_free_dof++); + } + + // for the quads + if (dof_handler.get_fe().dofs_per_quad > 0) + for (unsigned int q=0; q::quads_per_cell; ++q) + { + const typename DoFHandler<3,spacedim>::quad_iterator + quad = cell->quad(q); + + // distribute dofs if necessary: + // check whether quad dof is already + // numbered (check only first dof) + if (quad->dof_index(0) == DoFHandler<3,spacedim>::invalid_dof_index) + // if not: distribute dofs + for (unsigned int d=0; dset_dof_index (d, next_free_dof++); + } + + + // dofs of hex + if (dof_handler.get_fe().dofs_per_hex > 0) + for (unsigned int d=0; dset_dof_index (d, next_free_dof++); + + + // note that this cell has been + // processed + cell->set_user_flag (); + + return next_free_dof; + } + + + /** + * Distribute degrees of + * freedom on all cells, or + * on cells with the + * correct subdomain_id if + * the corresponding + * argument is not equal to + * types::invalid_subdomain_id. Return + * the total number of dofs + * returned. + */ + template + static + unsigned int + distribute_dofs (const unsigned int offset, + const unsigned int subdomain_id, + DoFHandler &dof_handler) + { + const dealii::Triangulation & tria + = dof_handler.get_tria(); + Assert (tria.n_levels() > 0, ExcMessage("Empty triangulation")); + + // Clear user flags because we will + // need them. But first we save + // them and make sure that we + // restore them later such that at + // the end of this function the + // Triangulation will be in the + // same state as it was at the + // beginning of this function. + std::vector user_flags; + tria.save_user_flags(user_flags); + const_cast &>(tria).clear_user_flags (); + + unsigned int next_free_dof = offset; + typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + + for (; cell != endc; ++cell) + if ((subdomain_id == types::invalid_subdomain_id) + || + (cell->subdomain_id() == subdomain_id)) + next_free_dof + = Implementation::distribute_dofs_on_cell (dof_handler, cell, next_free_dof); + + // update the cache used + // for cell dof indices + for (typename DoFHandler::cell_iterator + cell = dof_handler.begin(); cell != dof_handler.end(); ++cell) + if (cell->subdomain_id() != types::artificial_subdomain_id) + cell->update_cell_dof_indices_cache (); + + // finally restore the user flags + const_cast &>(tria).load_user_flags(user_flags); + + return next_free_dof; + } + + +/* --------------------- renumber_dofs functionality ---------------- */ + + + /** + * Implementation of the + * general template of same + * name. + * + * If the second argument + * has any elements set, + * elements of the then the + * vector of new numbers do + * not relate to the old + * DoF number but instead + * to the index of the old + * DoF number within the + * set of locally owned + * DoFs. + */ + template + static + void + renumber_dofs (const std::vector &new_numbers, + const IndexSet &, + DoFHandler<1,spacedim> &dof_handler, + const bool check_validity) + { + // note that we can not use cell + // iterators in this function since + // then we would renumber the dofs on + // the interface of two cells more + // than once. Anyway, this way it's + // not only more correct but also + // faster; note, however, that dof + // numbers may be invalid_dof_index, + // namely when the appropriate + // vertex/line/etc is unused + for (std::vector::iterator + i=dof_handler.vertex_dofs.begin(); + i!=dof_handler.vertex_dofs.end(); ++i) + if (*i != DoFHandler<1,spacedim>::invalid_dof_index) + *i = new_numbers[*i]; + else if (check_validity) + // if index is + // invalid_dof_index: + // check if this one + // really is unused + Assert (dof_handler.get_tria() + .vertex_used((i-dof_handler.vertex_dofs.begin()) / + dof_handler.selected_fe->dofs_per_vertex) + == false, + ExcInternalError ()); + + for (unsigned int level=0; level::iterator + i=dof_handler.levels[level]->lines.dofs.begin(); + i!=dof_handler.levels[level]->lines.dofs.end(); ++i) + if (*i != DoFHandler<1,spacedim>::invalid_dof_index) + *i = new_numbers[*i]; + + // update the cache + // used for cell dof + // indices + for (typename DoFHandler<1,spacedim>::cell_iterator + cell = dof_handler.begin(); + cell != dof_handler.end(); ++cell) + cell->update_cell_dof_indices_cache (); + } + + + + template + static + void + renumber_dofs (const std::vector &new_numbers, + const IndexSet &indices, + DoFHandler<2,spacedim> &dof_handler, + const bool check_validity) + { + // note that we can not use cell + // iterators in this function since + // then we would renumber the dofs on + // the interface of two cells more + // than once. Anyway, this way it's + // not only more correct but also + // faster; note, however, that dof + // numbers may be invalid_dof_index, + // namely when the appropriate + // vertex/line/etc is unused + for (std::vector::iterator + i=dof_handler.vertex_dofs.begin(); + i!=dof_handler.vertex_dofs.end(); ++i) + if (*i != DoFHandler<2,spacedim>::invalid_dof_index) + *i = (indices.n_elements() == 0)? + (new_numbers[*i]) : + (new_numbers[indices.index_within_set(*i)]); + else if (check_validity) + // if index is invalid_dof_index: + // check if this one really is + // unused + Assert (dof_handler.get_tria() + .vertex_used((i-dof_handler.vertex_dofs.begin()) / + dof_handler.selected_fe->dofs_per_vertex) + == false, + ExcInternalError ()); + + for (std::vector::iterator + i=dof_handler.faces->lines.dofs.begin(); + i!=dof_handler.faces->lines.dofs.end(); ++i) + if (*i != DoFHandler<2,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + + for (unsigned int level=0; level::iterator + i=dof_handler.levels[level]->quads.dofs.begin(); + i!=dof_handler.levels[level]->quads.dofs.end(); ++i) + if (*i != DoFHandler<2,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + } + + // update the cache + // used for cell dof + // indices + for (typename DoFHandler<2,spacedim>::cell_iterator + cell = dof_handler.begin(); + cell != dof_handler.end(); ++cell) + cell->update_cell_dof_indices_cache (); + } + + + template + static + void + renumber_dofs (const std::vector &new_numbers, + const IndexSet &indices, + DoFHandler<3,spacedim> &dof_handler, + const bool check_validity) + { + // note that we can not use cell + // iterators in this function since + // then we would renumber the dofs on + // the interface of two cells more + // than once. Anyway, this way it's + // not only more correct but also + // faster; note, however, that dof + // numbers may be invalid_dof_index, + // namely when the appropriate + // vertex/line/etc is unused + for (std::vector::iterator + i=dof_handler.vertex_dofs.begin(); + i!=dof_handler.vertex_dofs.end(); ++i) + if (*i != DoFHandler<3,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + else if (check_validity) + // if index is invalid_dof_index: + // check if this one really is + // unused + Assert (dof_handler.get_tria() + .vertex_used((i-dof_handler.vertex_dofs.begin()) / + dof_handler.selected_fe->dofs_per_vertex) + == false, + ExcInternalError ()); + + for (std::vector::iterator + i=dof_handler.faces->lines.dofs.begin(); + i!=dof_handler.faces->lines.dofs.end(); ++i) + if (*i != DoFHandler<3,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + for (std::vector::iterator + i=dof_handler.faces->quads.dofs.begin(); + i!=dof_handler.faces->quads.dofs.end(); ++i) + if (*i != DoFHandler<3,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + + for (unsigned int level=0; level::iterator + i=dof_handler.levels[level]->hexes.dofs.begin(); + i!=dof_handler.levels[level]->hexes.dofs.end(); ++i) + if (*i != DoFHandler<3,spacedim>::invalid_dof_index) + *i = ((indices.n_elements() == 0) ? + new_numbers[*i] : + new_numbers[indices.index_within_set(*i)]); + } + + // update the cache + // used for cell dof + // indices + for (typename DoFHandler<3,spacedim>::cell_iterator + cell = dof_handler.begin(); + cell != dof_handler.end(); ++cell) + cell->update_cell_dof_indices_cache (); + } + }; + + + +/* --------------------- class PolicyBase ---------------- */ + + template + PolicyBase::~PolicyBase () + {} + + +/* --------------------- class Sequential ---------------- */ + + + template + NumberCache + Sequential:: + distribute_dofs (const unsigned int offset, + DoFHandler &dof_handler) const + { + const unsigned int n_dofs = + Implementation::distribute_dofs (offset, + types::invalid_subdomain_id, + dof_handler); + + // now set the elements of the + // number cache appropriately + NumberCache number_cache; + number_cache.n_global_dofs = n_dofs; + number_cache.n_locally_owned_dofs = number_cache.n_global_dofs; + + number_cache.locally_owned_dofs + = IndexSet (number_cache.n_global_dofs); + number_cache.locally_owned_dofs.add_range (0, + number_cache.n_global_dofs); + + number_cache.n_locally_owned_dofs_per_processor + = std::vector (1, + number_cache.n_global_dofs); + + number_cache.locally_owned_dofs_per_processor + = std::vector (1, + number_cache.locally_owned_dofs); + return number_cache; + } + + + + template + NumberCache + Sequential:: + renumber_dofs (const std::vector &new_numbers, + dealii::DoFHandler &dof_handler) const + { + Implementation::renumber_dofs (new_numbers, IndexSet(0), + dof_handler, true); + + // in the sequential case, + // the number cache should + // not have changed but we + // have to set the elements + // of the structure + // appropriately anyway + NumberCache number_cache; + number_cache.n_global_dofs = dof_handler.n_dofs(); + number_cache.n_locally_owned_dofs = number_cache.n_global_dofs; + + number_cache.locally_owned_dofs + = IndexSet (number_cache.n_global_dofs); + number_cache.locally_owned_dofs.add_range (0, + number_cache.n_global_dofs); + + number_cache.n_locally_owned_dofs_per_processor + = std::vector (1, + number_cache.n_global_dofs); + + number_cache.locally_owned_dofs_per_processor + = std::vector (1, + number_cache.locally_owned_dofs); + return number_cache; + } + + + +/* --------------------- class ParallelDistributed ---------------- */ + +#ifdef DEAL_II_USE_P4EST + + namespace + { + template + struct types + { + + /** + * A list of tree+quadrant and + * their dof indices. dofs is of + * the form num_dofindices of + * quadrant 0, followed by + * num_dofindices indices, + * num_dofindices of quadrant 1, + * ... + */ + struct cellinfo + { + std::vector tree_index; + std::vector::quadrant> quadrants; + std::vector dofs; + + unsigned int bytes_for_buffer () const + { + return (sizeof(unsigned int) + + tree_index.size() * sizeof(unsigned int) + + quadrants.size() * sizeof(typename dealii::internal::p4est + ::types::quadrant) + + dofs.size() * sizeof(unsigned int)); + } + + void pack_data (std::vector &buffer) const + { + buffer.resize(bytes_for_buffer()); + + char * ptr = &buffer[0]; + + const unsigned int num_cells = tree_index.size(); + std::memcpy(ptr, &num_cells, sizeof(unsigned int)); + ptr += sizeof(unsigned int); + + std::memcpy(ptr, + &tree_index[0], + num_cells*sizeof(unsigned int)); + ptr += num_cells*sizeof(unsigned int); + + std::memcpy(ptr, + &quadrants[0], + num_cells * sizeof(typename dealii::internal::p4est:: + types::quadrant)); + ptr += num_cells*sizeof(typename dealii::internal::p4est::types:: + quadrant); + + std::memcpy(ptr, + &dofs[0], + dofs.size() * sizeof(unsigned int)); + ptr += dofs.size() * sizeof(unsigned int); + + Assert (ptr == &buffer[buffer.size()], + ExcInternalError()); + + } + }; + }; + + + + template + void + fill_dofindices_recursively (const typename parallel::distributed::Triangulation & tria, + const unsigned int tree_index, + const typename DoFHandler::cell_iterator &dealii_cell, + const typename dealii::internal::p4est::types::quadrant &p4est_cell, + const std::map > &vertices_with_ghost_neighbors, + std::map::cellinfo> &needs_to_get_cell) + { + // see if we have to + // recurse... + if (dealii_cell->has_children()) + { + typename dealii::internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + internal::p4est::init_quadrant_children(p4est_cell, p4est_child); + + + for (unsigned int c=0;c::max_children_per_cell; ++c) + fill_dofindices_recursively(tria, + tree_index, + dealii_cell->child(c), + p4est_child[c], + vertices_with_ghost_neighbors, + needs_to_get_cell); + return; + } + + // we're at a leaf cell. see if + // the cell is flagged as + // interesting. note that we + // have only flagged our own + // cells before + if (dealii_cell->user_flag_set() && !dealii_cell->is_ghost()) + { + Assert (!dealii_cell->is_artificial(), ExcInternalError()); + + // check each vertex if + // it is interesting and + // push dofindices if yes + std::set send_to; + for (unsigned int v=0; v::vertices_per_cell; ++v) + { + const std::map >::const_iterator + neighbor_subdomains_of_vertex + = vertices_with_ghost_neighbors.find (dealii_cell->vertex_index(v)); + + if (neighbor_subdomains_of_vertex == + vertices_with_ghost_neighbors.end()) + continue; + + Assert(neighbor_subdomains_of_vertex->second.size()!=0, + ExcInternalError()); + + send_to.insert(neighbor_subdomains_of_vertex->second.begin(), + neighbor_subdomains_of_vertex->second.end()); + } + + if (send_to.size() > 0) + { + // this cell's dof_indices + // need to be sent to + // someone + std::vector + local_dof_indices (dealii_cell->get_fe().dofs_per_cell); + dealii_cell->get_dof_indices (local_dof_indices); + + for (std::set::iterator it=send_to.begin(); + it!=send_to.end();++it) + { + const dealii::types::subdomain_id_t subdomain = *it; + + // get an iterator + // to what needs to + // be sent to that + // subdomain (if + // already exists), + // or create such + // an object + typename std::map::cellinfo>::iterator + p + = needs_to_get_cell.insert (std::make_pair(subdomain, + typename types::cellinfo())) + .first; + + p->second.tree_index.push_back(tree_index); + p->second.quadrants.push_back(p4est_cell); + + p->second.dofs.push_back(dealii_cell->get_fe().dofs_per_cell); + p->second.dofs.insert(p->second.dofs.end(), + local_dof_indices.begin(), + local_dof_indices.end()); + + } + } + } + } + + + template + void + set_dofindices_recursively ( + const parallel::distributed::Triangulation &tria, + const typename dealii::internal::p4est::types::quadrant &p4est_cell, + const typename DoFHandler::cell_iterator &dealii_cell, + const typename dealii::internal::p4est::types::quadrant &quadrant, + unsigned int * dofs) + { + if (internal::p4est::quadrant_is_equal(p4est_cell, quadrant)) + { + Assert(!dealii_cell->has_children(), ExcInternalError()); + Assert(dealii_cell->is_ghost(), ExcInternalError()); + + // update dof indices of cell + std::vector + dof_indices (dealii_cell->get_fe().dofs_per_cell); + dealii_cell->update_cell_dof_indices_cache(); + dealii_cell->get_dof_indices(dof_indices); + + bool complete = true; + for (unsigned int i=0;i::invalid_dof_index) + { + Assert((dof_indices[i] == + (DoFHandler::invalid_dof_index)) + || + (dof_indices[i]==dofs[i]), + ExcInternalError()); + dof_indices[i]=dofs[i]; + } + else + complete=false; + + if (!complete) + const_cast + ::cell_iterator &> + (dealii_cell)->set_user_flag(); + else + const_cast + ::cell_iterator &> + (dealii_cell)->clear_user_flag(); + + const_cast + ::cell_iterator &> + (dealii_cell)->set_dof_indices(dof_indices); + + return; + } + + if (! dealii_cell->has_children()) + return; + + if (! internal::p4est::quadrant_is_ancestor (p4est_cell, quadrant)) + return; + + typename dealii::internal::p4est::types::quadrant + p4est_child[GeometryInfo::max_children_per_cell]; + internal::p4est::init_quadrant_children(p4est_cell, p4est_child); + + for (unsigned int c=0; c::max_children_per_cell; ++c) + set_dofindices_recursively (tria, p4est_child[c], + dealii_cell->child(c), + quadrant, dofs); + } + + /** + * Return a vector in which, + * for every vertex index, we + * mark whether a locally owned + * cell is adjacent. + */ + template + std::vector + mark_locally_active_vertices (const parallel::distributed::Triangulation &triangulation) + { + std::vector locally_active_vertices (triangulation.n_vertices(), + false); + + for (typename dealii::Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell) + if (cell->subdomain_id() == triangulation.locally_owned_subdomain()) + for (unsigned int v=0; v::vertices_per_cell; ++v) + locally_active_vertices[cell->vertex_index(v)] = true; + + return locally_active_vertices; + } + + + + template + void + communicate_dof_indices_on_marked_cells + (const DoFHandler<1,spacedim> &, + const std::map > &, + const std::vector &, + const std::vector &) + { + Assert (false, ExcNotImplemented()); + } + + + + template + void + communicate_dof_indices_on_marked_cells + (const DoFHandler &dof_handler, + const std::map > &vertices_with_ghost_neighbors, + const std::vector &coarse_cell_to_p4est_tree_permutation, + const std::vector &p4est_tree_to_coarse_cell_permutation) + { +#ifndef DEAL_II_USE_P4EST + (void)vertices_with_ghost_neighbors; + Assert (false, ExcNotImplemented()); +#else + + const parallel::distributed::Triangulation< dim, spacedim > * tr + = (dynamic_cast*> + (&dof_handler.get_tria())); + Assert (tr != 0, ExcInternalError()); + + // now collect cells and their + // dof_indices for the + // interested neighbors + typedef + std::map::cellinfo> + cellmap_t; + cellmap_t needs_to_get_cells; + + for (typename DoFHandler::cell_iterator + cell = dof_handler.begin(0); + cell != dof_handler.end(0); + ++cell) + { + typename dealii::internal::p4est::types::quadrant p4est_coarse_cell; + internal::p4est::init_coarse_quadrant(p4est_coarse_cell); + + fill_dofindices_recursively + (*tr, + coarse_cell_to_p4est_tree_permutation[cell->index()], + cell, + p4est_coarse_cell, + vertices_with_ghost_neighbors, + needs_to_get_cells); + } + + + + //sending + std::vector > sendbuffers (needs_to_get_cells.size()); + std::vector >::iterator buffer = sendbuffers.begin(); + std::vector requests (needs_to_get_cells.size()); + + unsigned int idx=0; + + for (typename cellmap_t::iterator it=needs_to_get_cells.begin(); + it!=needs_to_get_cells.end(); + ++it, ++buffer, ++idx) + { + const unsigned int num_cells = it->second.tree_index.size(); + + Assert(num_cells==it->second.quadrants.size(), ExcInternalError()); + Assert(num_cells>0, ExcInternalError()); + + // pack all the data into + // the buffer for this + // recipient and send + // it. keep data around + // till we can make sure + // that the packet has been + // received + it->second.pack_data (*buffer); + MPI_Isend(&(*buffer)[0], buffer->size(), + MPI_BYTE, it->first, + 123, tr->get_communicator(), &requests[idx]); + } + + + // mark all own cells, that miss some + // dof_data and collect the neighbors + // that are going to send stuff to us + std::set senders; + { + std::vector local_dof_indices; + typename DoFHandler::active_cell_iterator + cell, endc = dof_handler.end(); + + for (cell = dof_handler.begin_active(); cell != endc; ++cell) + if (!cell->is_artificial()) + { + if (cell->is_ghost()) + { + if (cell->user_flag_set()) + senders.insert(cell->subdomain_id()); + } + else + { + local_dof_indices.resize (cell->get_fe().dofs_per_cell); + cell->get_dof_indices (local_dof_indices); + if (local_dof_indices.end() != + std::find (local_dof_indices.begin(), + local_dof_indices.end(), + DoFHandler::invalid_dof_index)) + cell->set_user_flag(); + else + cell->clear_user_flag(); + } + + } + } + + + //* 5. receive ghostcelldata + std::vector receive; + typename types::cellinfo cellinfo; + for (unsigned int i=0;iget_communicator(), &status); + MPI_Get_count(&status, MPI_BYTE, &len); + receive.resize(len); + + char * ptr = &receive[0]; + MPI_Recv(ptr, len, MPI_BYTE, status.MPI_SOURCE, status.MPI_TAG, + tr->get_communicator(), &status); + + unsigned int cells; + memcpy(&cells, ptr, sizeof(unsigned int)); + ptr+=sizeof(unsigned int); + + //reinterpret too evil? + unsigned int * treeindex=reinterpret_cast(ptr); + ptr+=cells*sizeof(unsigned int); + typename dealii::internal::p4est::types::quadrant * quadrant + =reinterpret_cast::quadrant *>(ptr); + ptr+=cells*sizeof(typename dealii::internal::p4est::types::quadrant); + unsigned int * dofs=reinterpret_cast(ptr); + + for (unsigned int c=0;c::cell_iterator + cell (&dof_handler.get_tria(), + 0, + p4est_tree_to_coarse_cell_permutation[treeindex[c]], + &dof_handler); + + typename dealii::internal::p4est::types::quadrant p4est_coarse_cell; + internal::p4est::init_coarse_quadrant(p4est_coarse_cell); + + Assert(cell->get_fe().dofs_per_cell==dofs[0], ExcInternalError()); + + set_dofindices_recursively (*tr, + p4est_coarse_cell, + cell, + quadrant[c], + (dofs+1)); + } + } + + // complete all sends, so that we can + // safely destroy the buffers. + MPI_Waitall(requests.size(), &requests[0], MPI_STATUSES_IGNORE); + + +#ifdef DEBUG + { + //check all msgs got sent and received + unsigned int sum_send=0; + unsigned int sum_recv=0; + unsigned int sent=needs_to_get_cells.size(); + unsigned int recv=senders.size(); + + MPI_Reduce(&sent, &sum_send, 1, MPI_INT, MPI_SUM, 0, tr->get_communicator()); + MPI_Reduce(&recv, &sum_recv, 1, MPI_INT, MPI_SUM, 0, tr->get_communicator()); + Assert(sum_send==sum_recv, ExcInternalError()); + } +#endif + + //update dofindices + { + typename DoFHandler::active_cell_iterator + cell, endc = dof_handler.end(); + + for (cell = dof_handler.begin_active(); cell != endc; ++cell) + if (!cell->is_artificial()) + cell->update_cell_dof_indices_cache(); + } + + // important, so that sends between two + // calls to this function are not mixed + // up. + // + // this is necessary because above we + // just see if there are messages and + // then receive them, without + // discriminating where they come from + // and whether they were sent in phase + // 1 or 2. the need for a global + // communication step like this barrier + // could be avoided by receiving + // messages specifically from those + // processors from which we expect + // messages, and by using different + // tags for phase 1 and 2 + MPI_Barrier(tr->get_communicator()); +#endif + } + } + +#endif // DEAL_II_USE_P4EST + + + + template + NumberCache + ParallelDistributed:: + distribute_dofs (const unsigned int offset, + DoFHandler &dof_handler) const + { + Assert(offset==0, ExcNotImplemented()); + NumberCache number_cache; + +#ifndef DEAL_II_USE_P4EST + (void)fe; + Assert (false, ExcNotImplemented()); +#else + + + parallel::distributed::Triangulation< dim, spacedim > * tr + = (dynamic_cast*> + (const_cast*> + (&dof_handler.get_tria()))); + Assert (tr != 0, ExcInternalError()); + + const unsigned int + n_cpus = Utilities::System::get_n_mpi_processes (tr->get_communicator()); + + //* 1. distribute on own + //* subdomain + const unsigned int n_initial_local_dofs = + Implementation::distribute_dofs (0, tr->locally_owned_subdomain(), + dof_handler); + + //* 2. iterate over ghostcells and + //kill dofs that are not owned + //by us + std::vector renumbering(n_initial_local_dofs); + for (unsigned int i=0; i local_dof_indices; + + typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + + for (; cell != endc; ++cell) + if (cell->is_ghost() && + (cell->subdomain_id() < tr->locally_owned_subdomain())) + { + // we found a + // neighboring ghost + // cell whose subdomain + // is "stronger" than + // our own subdomain + + // delete all dofs that + // live there and that + // we have previously + // assigned a number to + // (i.e. the ones on + // the interface) + local_dof_indices.resize (cell->get_fe().dofs_per_cell); + cell->get_dof_indices (local_dof_indices); + for (unsigned int i=0; iget_fe().dofs_per_cell; ++i) + if (local_dof_indices[i] != DoFHandler::invalid_dof_index) + renumbering[local_dof_indices[i]] + = DoFHandler::invalid_dof_index; + } + } + + + // make indices consecutive + number_cache.n_locally_owned_dofs = 0; + for (std::vector::iterator it=renumbering.begin(); + it!=renumbering.end(); ++it) + if (*it != DoFHandler::invalid_dof_index) + *it = number_cache.n_locally_owned_dofs++; + + //* 3. communicate local dofcount and + //shift ids to make them unique + number_cache.n_locally_owned_dofs_per_processor.resize(n_cpus); + + MPI_Allgather ( &number_cache.n_locally_owned_dofs, + 1, MPI_INT, + &number_cache.n_locally_owned_dofs_per_processor[0], + 1, MPI_INT, + tr->get_communicator()); + + const unsigned int + shift = std::accumulate( &number_cache + .n_locally_owned_dofs_per_processor[0], + &number_cache + .n_locally_owned_dofs_per_processor + [tr->locally_owned_subdomain()], + 0 ); + for (std::vector::iterator it=renumbering.begin(); + it!=renumbering.end(); ++it) + if (*it != DoFHandler::invalid_dof_index) + (*it) += shift; + + // now re-enumerate all dofs to + // this shifted and condensed + // numbering form. we renumber + // some dofs as invalid, so + // choose the nocheck-version. + Implementation::renumber_dofs (renumbering, IndexSet(0), + dof_handler, false); + + // now a little bit of + // housekeeping + number_cache.n_global_dofs + = std::accumulate( &number_cache + .n_locally_owned_dofs_per_processor[0], + &number_cache + .n_locally_owned_dofs_per_processor[n_cpus], + 0 ); + + number_cache.locally_owned_dofs = IndexSet(number_cache.n_global_dofs); + number_cache.locally_owned_dofs + .add_range(shift, + shift+number_cache.n_locally_owned_dofs); + + // fill global_dof_indexsets + number_cache.locally_owned_dofs_per_processor.resize(n_cpus); + { + unsigned int lshift = 0; + for (unsigned int i=0;ilocally_owned_subdomain()].n_elements() + == + number_cache.n_locally_owned_dofs, + ExcInternalError()); + Assert(!number_cache.locally_owned_dofs_per_processor + [tr->locally_owned_subdomain()].n_elements() + || + number_cache.locally_owned_dofs_per_processor + [tr->locally_owned_subdomain()].nth_index_in_set(0) + == shift, + ExcInternalError()); + + //* 4. send dofids of cells that are + //ghostcells on other machines + + std::vector user_flags; + tr->save_user_flags(user_flags); + tr->clear_user_flags (); + + //mark all own cells for transfer + typename DoFHandler::active_cell_iterator + cell, endc = dof_handler.end(); + for (cell = dof_handler.begin_active(); cell != endc; ++cell) + if (!cell->is_artificial()) + cell->set_user_flag(); + + //mark the vertices we are interested + //in, i.e. belonging to own and marked cells + const std::vector locally_active_vertices + = mark_locally_active_vertices (*tr); + + // add each ghostcells' + // subdomain to the vertex and + // keep track of interesting + // neighbors + std::map > + vertices_with_ghost_neighbors; + for (typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(); + cell != dof_handler.end(); ++cell) + if (cell->is_ghost ()) + for (unsigned int v=0; v::vertices_per_cell; ++v) + if (locally_active_vertices[cell->vertex_index(v)]) + vertices_with_ghost_neighbors[cell->vertex_index(v)] + .insert (cell->subdomain_id()); + + + /* Send and receive cells. After this, + only the local cells are marked, + that received new data. This has to + be communicated in a second + communication step. */ + communicate_dof_indices_on_marked_cells (dof_handler, + vertices_with_ghost_neighbors, + tr->coarse_cell_to_p4est_tree_permutation, + tr->p4est_tree_to_coarse_cell_permutation); + + communicate_dof_indices_on_marked_cells (dof_handler, + vertices_with_ghost_neighbors, + tr->coarse_cell_to_p4est_tree_permutation, + tr->p4est_tree_to_coarse_cell_permutation); + + tr->load_user_flags(user_flags); + +#ifdef DEBUG + //check that we are really done + { + std::vector local_dof_indices; + typename DoFHandler::active_cell_iterator + cell, endc = dof_handler.end(); + + for (cell = dof_handler.begin_active(); cell != endc; ++cell) + if (!cell->is_artificial()) + { + local_dof_indices.resize (cell->get_fe().dofs_per_cell); + cell->get_dof_indices (local_dof_indices); + if (local_dof_indices.end() != + std::find (local_dof_indices.begin(), + local_dof_indices.end(), + DoFHandler::invalid_dof_index)) + { + if (cell->is_ghost()) + { + Assert(false, ExcMessage ("Not a ghost cell")); + } + else + { + Assert(false, ExcMessage ("Not one of our own cells")); + } + } + } + } +#endif // DEBUG +#endif // DEAL_II_USE_P4EST + + return number_cache; + } + + + + template + NumberCache + ParallelDistributed:: + renumber_dofs (const std::vector &new_numbers, + dealii::DoFHandler &dof_handler) const + { + Assert (new_numbers.size() == dof_handler.locally_owned_dofs().n_elements(), + ExcInternalError()); + + NumberCache number_cache; + +#ifndef DEAL_II_USE_P4EST + Assert (false, ExcNotImplemented()); +#else + + + //calculate new IndexSet. First try + //to find out if the new indices are + //contiguous blocks. This avoids + //inserting each index individually + //into the IndexSet, which is slow. + //If we own no DoFs, we still need to + //go through this function, but we + //can skip this calculation. + + number_cache.locally_owned_dofs = IndexSet (dof_handler.n_dofs()); + if (dof_handler.locally_owned_dofs().n_elements()>0) + { + std::vector::const_iterator it = new_numbers.begin(); + const unsigned int n_blocks = dof_handler.get_fe().n_blocks(); + std::vector > block_indices(n_blocks); + block_indices[0].first = *it++; + block_indices[0].second = 1; + unsigned int current_block = 0, n_filled_blocks = 1; + for ( ; it != new_numbers.end(); ++it) + { + bool done = false; + + // search from the current block onwards + // whether the next index is shifted by one + // from the previous one. + for (unsigned int i=0; i local_dof_indices; + + typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + + for (; cell != endc; ++cell) + if (!cell->is_artificial()) + { + local_dof_indices.resize (cell->get_fe().dofs_per_cell); + cell->get_dof_indices (local_dof_indices); + for (unsigned int i=0; iget_fe().dofs_per_cell; ++i) + { + if (local_dof_indices[i] == DoFHandler::invalid_dof_index) + continue; + + if (!dof_handler.locally_owned_dofs().is_element(local_dof_indices[i])) + { + //this DoF is not owned + //by us, so set it to + //invalid. + local_dof_indices[i] + = DoFHandler::invalid_dof_index; + } + } + + cell->set_dof_indices (local_dof_indices); + } + } + + + // renumber. Skip when there is + // nothing to do because we own no + // DoF. + if (dof_handler.locally_owned_dofs().n_elements() > 0) + Implementation::renumber_dofs (new_numbers, + dof_handler.locally_owned_dofs(), + dof_handler, + false); + + // communication + { + parallel::distributed::Triangulation< dim, spacedim > * tr + = (dynamic_cast*> + (const_cast*> + (&dof_handler.get_tria()))); + Assert (tr != 0, ExcInternalError()); + + std::vector user_flags; + tr->save_user_flags(user_flags); + tr->clear_user_flags (); + + //mark all own cells for transfer + typename DoFHandler::active_cell_iterator + cell, endc = dof_handler.end(); + for (cell = dof_handler.begin_active(); cell != endc; ++cell) + if (!cell->is_artificial()) + cell->set_user_flag(); + //mark the vertices we are interested + //in, i.e. belonging to own and marked cells + const std::vector locally_active_vertices + = mark_locally_active_vertices (*tr); + + // add each ghostcells' + // subdomain to the vertex and + // keep track of interesting + // neighbors + std::map > + vertices_with_ghost_neighbors; + for (typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(); + cell != dof_handler.end(); ++cell) + if (cell->is_ghost ()) + for (unsigned int v=0; v::vertices_per_cell; ++v) + if (locally_active_vertices[cell->vertex_index(v)]) + vertices_with_ghost_neighbors[cell->vertex_index(v)] + .insert (cell->subdomain_id()); + + // Send and receive cells. After this, only + // the local cells are marked, that received + // new data. This has to be communicated in a + // second communication step. + communicate_dof_indices_on_marked_cells (dof_handler, + vertices_with_ghost_neighbors, + tr->coarse_cell_to_p4est_tree_permutation, + tr->p4est_tree_to_coarse_cell_permutation); + + communicate_dof_indices_on_marked_cells (dof_handler, + vertices_with_ghost_neighbors, + tr->coarse_cell_to_p4est_tree_permutation, + tr->p4est_tree_to_coarse_cell_permutation); + + + // * Create global_dof_indexsets by + // transfering our own owned_dofs to + // every other machine. + const unsigned int n_cpus = Utilities::System:: + get_n_mpi_processes (tr->get_communicator()); + + // Serialize our IndexSet and + // determine size. + std::ostringstream oss; + number_cache.locally_owned_dofs.block_write(oss); + std::string oss_str=oss.str(); + std::vector my_data(oss_str.begin(), oss_str.end()); + unsigned int my_size = my_data.size(); + + // determine maximum size of IndexSet + unsigned int max_size; + + MPI_Allreduce(&my_size, &max_size, 1, MPI_INT, MPI_MAX, tr->get_communicator()); + + // as we are reading past the end, we + // need to increase the size of the + // local buffer. This is filled with + // zeros. + my_data.resize(max_size); + + std::vector buffer(max_size*n_cpus); + MPI_Allgather(&my_data[0], max_size, MPI_BYTE, + &buffer[0], max_size, MPI_BYTE, + tr->get_communicator()); + + number_cache.locally_owned_dofs_per_processor.resize (n_cpus); + number_cache.n_locally_owned_dofs_per_processor.resize (n_cpus); + for (unsigned int i=0;iload_user_flags(user_flags); + } +#endif + + return number_cache; + } + } + } +} + + + + +/*-------------- Explicit Instantiations -------------------------------*/ + +namespace internal +{ + namespace DoFHandler + { + namespace Policy + { + template class PolicyBase; + template class Sequential; + template class ParallelDistributed; + +#if deal_II_dimension==1 || deal_II_dimension==2 + template class PolicyBase; + template class Sequential; + template class ParallelDistributed; +#endif + } + } +} + + +DEAL_II_NAMESPACE_CLOSE diff --git a/deal.II/deal.II/source/dofs/dof_renumbering.cc b/deal.II/deal.II/source/dofs/dof_renumbering.cc index 990948db7e..c6b16a87d7 100644 --- a/deal.II/deal.II/source/dofs/dof_renumbering.cc +++ b/deal.II/deal.II/source/dofs/dof_renumbering.cc @@ -1,5 +1,5 @@ //--------------------------------------------------------------------------- -// $id$ +// $Id$ // Version: $name$ // // Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors @@ -78,6 +78,8 @@ namespace DoFRenumbering class WrapDoFIterator : private T { public: + typedef typename T::AccessorType AccessorType; + WrapDoFIterator (const T& t) : T(t) {} void get_dof_indices (std::vector& v) const @@ -102,6 +104,8 @@ namespace DoFRenumbering class WrapMGDoFIterator : private T { public: + typedef typename T::AccessorType AccessorType; + WrapMGDoFIterator (const T& t) : T(t) {} void get_dof_indices (std::vector& v) const @@ -505,6 +509,7 @@ namespace DoFRenumbering } + template void Cuthill_McKee (MGDoFHandler &dof_handler, const unsigned int level, @@ -532,12 +537,13 @@ namespace DoFRenumbering } + template void component_wise (DoFHandler &dof_handler, const std::vector &component_order_arg) { - std::vector renumbering (dof_handler.n_dofs(), + std::vector renumbering (dof_handler.n_locally_owned_dofs(), DoFHandler::invalid_dof_index); typedef @@ -554,10 +560,21 @@ namespace DoFRenumbering typename DoFHandler::cell_iterator>(renumbering, start, end, component_order_arg); - - if (result == 0) return; - - Assert (result == dof_handler.n_dofs(), + if (result == 0) + return; + + // verify that the last numbered + // degree of freedom is either + // equal to the number of degrees + // of freedom in total (the + // sequential case) or in the + // distributed case at least + // makes sense + Assert ((result == dof_handler.n_locally_owned_dofs()) + || + ((dof_handler.n_locally_owned_dofs() < dof_handler.n_dofs()) + && + (result <= dof_handler.n_dofs())), ExcRenumberingIncomplete()); dof_handler.renumber_dofs (renumbering); @@ -597,6 +614,7 @@ namespace DoFRenumbering } + template void component_wise (MGDoFHandler &dof_handler, @@ -630,6 +648,7 @@ namespace DoFRenumbering } + template void component_wise (MGDoFHandler &dof_handler, @@ -656,11 +675,10 @@ namespace DoFRenumbering - template unsigned int compute_component_wise (std::vector& new_indices, - ITERATOR& start, + const ITERATOR & start, const ENDITERATOR& end, const std::vector &component_order_arg) { @@ -674,10 +692,6 @@ namespace DoFRenumbering return 0; } -// Assert (new_indices.size() == start->dof_handler().n_dofs(), -// ExcDimensionMismatch(new_indices.size(), -// start->dof_handler.n_dofs())); - // Copy last argument into a // writable vector. std::vector component_order (component_order_arg); @@ -692,7 +706,7 @@ namespace DoFRenumbering Assert (component_order.size() == fe_collection.n_components(), ExcDimensionMismatch(component_order.size(), fe_collection.n_components())); - for (unsigned int i=0; i< component_order.size(); ++i) + for (unsigned int i=0; i > component_to_dof_map (fe_collection.n_components()); - for (;start!=end;++start) - { + for (ITERATOR cell=start; cell!=end; ++cell) + if (!cell->is_artificial() && !cell->is_ghost()) + { // on each cell: get dof indices // and insert them into the global // list using their component - const unsigned int fe_index = start->active_fe_index(); - const unsigned int dofs_per_cell =fe_collection[fe_index].dofs_per_cell; - local_dof_indices.resize (dofs_per_cell); - start.get_dof_indices (local_dof_indices); - for (unsigned int i=0; iactive_fe_index(); + const unsigned int dofs_per_cell =fe_collection[fe_index].dofs_per_cell; + local_dof_indices.resize (dofs_per_cell); + cell.get_dof_indices (local_dof_indices); + for (unsigned int i=0; iget_dof_handler().locally_owned_dofs().is_element(local_dof_indices[i])) + component_to_dof_map[component_list[fe_index][i]]. + push_back (local_dof_indices[i]); + } // now we've got all indices sorted // into buckets labelled with their @@ -793,25 +809,88 @@ namespace DoFRenumbering .erase (std::unique (component_to_dof_map[component].begin(), component_to_dof_map[component].end()), component_to_dof_map[component].end()); - }; + } + + // calculate the number of locally owned + // DoFs per bucket + const unsigned int n_buckets = fe_collection.n_components(); + std::vector shifts(n_buckets); + + if (const parallel::distributed::Triangulation * tria + = (dynamic_cast*> + (&start->get_dof_handler().get_tria()))) + { +#ifdef DEAL_II_USE_P4EST + std::vector local_dof_count(n_buckets); + + for (unsigned int c=0; c + all_dof_counts(fe_collection.n_components() * + Utilities::System::get_n_mpi_processes (tria->get_communicator())); + + MPI_Allgather ( &local_dof_count[0], n_buckets, MPI_INT, &all_dof_counts[0], + n_buckets, MPI_INT, tria->get_communicator()); + + for (unsigned int i=0; ilocally_owned_subdomain()+i] + == + local_dof_count[i], + ExcInternalError()); + + //calculate shifts + unsigned int cumulated = 0; + for (unsigned int c=0; clocally_owned_subdomain(); ++i) + shifts[c] += all_dof_counts[c+n_buckets*i]; + for (unsigned int i=0; iget_communicator()); ++i) + cumulated += all_dof_counts[c+n_buckets*i]; + } +#else + Assert (false, ExcInternalError()); +#endif + } + else + { + shifts[0] = 0; + for (unsigned int c=1; c::const_iterator begin_of_component = component_to_dof_map[component].begin(), end_of_component = component_to_dof_map[component].end(); + next_free_index = shifts[component]; + for (typename std::vector::const_iterator dof_index = begin_of_component; dof_index != end_of_component; ++dof_index) - new_indices[*dof_index] = next_free_index++; - }; + { + Assert (start->get_dof_handler().locally_owned_dofs() + .index_within_set(*dof_index) + < + new_indices.size(), + ExcInternalError()); + new_indices[start->get_dof_handler().locally_owned_dofs() + .index_within_set(*dof_index)] + = next_free_index++; + } + } return next_free_index; } @@ -1609,7 +1688,7 @@ namespace DoFRenumbering // first get the association of each dof // with a subdomain and determine the total // number of subdomain ids used - std::vector subdomain_association (n_dofs); + std::vector subdomain_association (n_dofs); DoFTools::get_subdomain_association (dof_handler, subdomain_association); const unsigned int n_subdomains diff --git a/deal.II/deal.II/source/dofs/dof_tools.cc b/deal.II/deal.II/source/dofs/dof_tools.cc index 2a85824601..c6fe150a67 100644 --- a/deal.II/deal.II/source/dofs/dof_tools.cc +++ b/deal.II/deal.II/source/dofs/dof_tools.cc @@ -79,9 +79,13 @@ DoFTools::make_sparsity_pattern (const DH &dof, // current cell is owned by the calling // processor. Otherwise, just continue. for (; cell!=endc; ++cell) - if ((subdomain_id == numbers::invalid_unsigned_int) - || - (subdomain_id == cell->subdomain_id())) + if (((subdomain_id == types::invalid_subdomain_id) + || + (subdomain_id == cell->subdomain_id())) + && + !cell->is_artificial() + && + !cell->is_ghost()) { const unsigned int dofs_per_cell = cell->get_fe().dofs_per_cell; dofs_on_this_cell.resize (dofs_per_cell); @@ -106,7 +110,7 @@ DoFTools::make_sparsity_pattern (const DH &dof, SparsityPattern &sparsity, const ConstraintMatrix &constraints, const bool keep_constrained_dofs, - const unsigned int subdomain_id) + const types::subdomain_id_t subdomain_id) { const unsigned int n_dofs = dof.n_dofs(); @@ -190,9 +194,13 @@ DoFTools::make_sparsity_pattern (const DH &dof, // current cell is owned by the calling // processor. Otherwise, just continue. for (; cell!=endc; ++cell) - if ((subdomain_id == numbers::invalid_unsigned_int) - || - (subdomain_id == cell->subdomain_id())) + if (((subdomain_id == types::invalid_subdomain_id) + || + (subdomain_id == cell->subdomain_id())) + && + !cell->is_artificial() + && + !cell->is_ghost()) { const unsigned int fe_index = cell->active_fe_index(); const unsigned int dofs_per_cell =fe_collection[fe_index].dofs_per_cell; @@ -1749,6 +1757,18 @@ namespace internal const unsigned int n_master_dofs = master_dofs.size (); const unsigned int n_slave_dofs = slave_dofs.size (); + // check for a couple + // conditions that happened + // in parallel distributed + // mode + for (unsigned int row=0; row!=n_slave_dofs; ++row) + Assert (slave_dofs[row] != numbers::invalid_unsigned_int, + ExcInternalError()); + for (unsigned int col=0; col!=n_master_dofs; ++col) + Assert (master_dofs[col] != numbers::invalid_unsigned_int, + ExcInternalError()); + + for (unsigned int row=0; row!=n_slave_dofs; ++row) if (constraints.is_constrained (slave_dofs[row]) == false) { @@ -1913,7 +1933,7 @@ namespace internal // currently not used but may be in the future: - + // static // void // make_hp_hanging_node_constraints (const dealii::MGDoFHandler<1,2> &, @@ -1986,116 +2006,126 @@ namespace internal typename DH::active_cell_iterator cell = dof_handler.begin_active(), endc = dof_handler.end(); for (; cell!=endc; ++cell) - for (unsigned int face=0; face::faces_per_cell; ++face) - if (cell->face(face)->has_children()) - { - // so now we've found a - // face of an active - // cell that has - // children. that means - // that there are - // hanging nodes here. - - // in any case, faces - // can have at most two - // active fe indices, - // but here the face - // can have only one - // (namely the same as - // that from the cell - // we're sitting on), - // and each of the - // children can have - // only one as - // well. check this - Assert (cell->face(face)->n_active_fe_indices() == 1, - ExcInternalError()); - Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) - == true, - ExcInternalError()); - for (unsigned int c=0; cface(face)->n_children(); ++c) - Assert (cell->face(face)->child(c)->n_active_fe_indices() == 1, + // artificial cells can at + // best neighbor ghost cells, + // but we're not interested + // in these interfaces + if (!cell->is_artificial ()) + for (unsigned int face=0; face::faces_per_cell; ++face) + if (cell->face(face)->has_children()) + { + // in any case, faces + // can have at most two + // active fe indices, + // but here the face + // can have only one + // (namely the same as + // that from the cell + // we're sitting on), + // and each of the + // children can have + // only one as + // well. check this + Assert (cell->face(face)->n_active_fe_indices() == 1, ExcInternalError()); + Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) + == true, + ExcInternalError()); + for (unsigned int c=0; cface(face)->n_children(); ++c) + if (!cell->neighbor_child_on_subface(face,c)->is_artificial()) + Assert (cell->face(face)->child(c)->n_active_fe_indices() == 1, + ExcInternalError()); - // right now, all that - // is implemented is - // the case that both - // sides use the same - // fe - for (unsigned int c=0; cface(face)->n_children(); ++c) - Assert (cell->face(face)->child(c) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcNotImplemented()); - - // ok, start up the work - const FiniteElement &fe = cell->get_fe(); - const unsigned int fe_index = cell->active_fe_index(); + // right now, all that + // is implemented is + // the case that both + // sides use the same + // fe + for (unsigned int c=0; cface(face)->n_children(); ++c) + if (!cell->neighbor_child_on_subface(face,c)->is_artificial()) + Assert (cell->face(face)->child(c) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcNotImplemented()); + + // ok, start up the work + const FiniteElement &fe = cell->get_fe(); + const unsigned int fe_index = cell->active_fe_index(); + + const unsigned int + n_dofs_on_mother = 2*fe.dofs_per_vertex + fe.dofs_per_line, + n_dofs_on_children = fe.dofs_per_vertex + 2*fe.dofs_per_line; + + dofs_on_mother.resize (n_dofs_on_mother); + dofs_on_children.resize (n_dofs_on_children); + + Assert(n_dofs_on_mother == fe.constraints().n(), + ExcDimensionMismatch(n_dofs_on_mother, + fe.constraints().n())); + Assert(n_dofs_on_children == fe.constraints().m(), + ExcDimensionMismatch(n_dofs_on_children, + fe.constraints().m())); + + const typename DH::line_iterator this_face = cell->face(face); + + // fill the dofs indices. Use same + // enumeration scheme as in + // @p{FiniteElement::constraints()} + unsigned int next_index = 0; + for (unsigned int vertex=0; vertex<2; ++vertex) + for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) + dofs_on_mother[next_index++] = this_face->vertex_dof_index(vertex,dof, + fe_index); + for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) + dofs_on_mother[next_index++] = this_face->dof_index(dof, fe_index); + AssertDimension (next_index, dofs_on_mother.size()); - const unsigned int - n_dofs_on_mother = 2*fe.dofs_per_vertex + fe.dofs_per_line, - n_dofs_on_children = fe.dofs_per_vertex + 2*fe.dofs_per_line; - - dofs_on_mother.resize (n_dofs_on_mother); - dofs_on_children.resize (n_dofs_on_children); - - Assert(n_dofs_on_mother == fe.constraints().n(), - ExcDimensionMismatch(n_dofs_on_mother, - fe.constraints().n())); - Assert(n_dofs_on_children == fe.constraints().m(), - ExcDimensionMismatch(n_dofs_on_children, - fe.constraints().m())); - - const typename DH::line_iterator this_face = cell->face(face); - - // fill the dofs indices. Use same - // enumeration scheme as in - // @p{FiniteElement::constraints()} - unsigned int next_index = 0; - for (unsigned int vertex=0; vertex<2; ++vertex) + next_index = 0; for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) - dofs_on_mother[next_index++] = this_face->vertex_dof_index(vertex,dof, - fe_index); - for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) - dofs_on_mother[next_index++] = this_face->dof_index(dof, fe_index); - AssertDimension (next_index, dofs_on_mother.size()); - - next_index = 0; - for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) - dofs_on_children[next_index++] - = this_face->child(0)->vertex_dof_index(1,dof,fe_index); - for (unsigned int child=0; child<2; ++child) - for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) dofs_on_children[next_index++] - = this_face->child(child)->dof_index(dof, fe_index); - AssertDimension (next_index, dofs_on_children.size()); + = this_face->child(0)->vertex_dof_index(1,dof,fe_index); + for (unsigned int child=0; child<2; ++child) + for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) + dofs_on_children[next_index++] + = this_face->child(child)->dof_index(dof, fe_index); + AssertDimension (next_index, dofs_on_children.size()); - // for each row in the constraint - // matrix for this line: - for (unsigned int row=0; row!=dofs_on_children.size(); ++row) - { - constraints.add_line (dofs_on_children[row]); - for (unsigned int i=0; i!=dofs_on_mother.size(); ++i) - constraints.add_entry (dofs_on_children[row], - dofs_on_mother[i], - fe.constraints()(row,i)); + // for each row in the constraint + // matrix for this line: + for (unsigned int row=0; row!=dofs_on_children.size(); ++row) + { + constraints.add_line (dofs_on_children[row]); + for (unsigned int i=0; i!=dofs_on_mother.size(); ++i) + constraints.add_entry (dofs_on_children[row], + dofs_on_mother[i], + fe.constraints()(row,i)); - constraints.set_inhomogeneity (dofs_on_children[row], 0.); - } - } - else - { - // this face has no - // children, but it - // could still be that - // it is shared by two - // cells that use a - // different fe index - Assert (cell->face(face)->n_active_fe_indices() == 1, - ExcNotImplemented()); - Assert (cell->face(face) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcInternalError()); - } + constraints.set_inhomogeneity (dofs_on_children[row], 0.); + } + } + else + { + // this face has no + // children, but it + // could still be + // that it is shared + // by two cells that + // use a different fe + // index. check a + // couple of things, + // but ignore the + // case that the + // neighbor is an + // artificial cell + if (!cell->at_boundary(face) && + !cell->neighbor(face)->is_artificial()) + { + Assert (cell->face(face)->n_active_fe_indices() == 1, + ExcNotImplemented()); + Assert (cell->face(face) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcInternalError()); + } + } } #endif @@ -2132,205 +2162,214 @@ namespace internal typename DH::active_cell_iterator cell = dof_handler.begin_active(), endc = dof_handler.end(); for (; cell!=endc; ++cell) - for (unsigned int face=0; face::faces_per_cell; ++face) - if (cell->face(face)->has_children()) - { - // first of all, make sure that - // we treat a case which is - // possible, i.e. either no dofs - // on the face at all or no - // anisotropic refinement - if (cell->get_fe().dofs_per_face == 0) - continue; - - Assert(cell->face(face)->refinement_case()==RefinementCase::isotropic_refinement, - ExcNotImplemented()); - - // so now we've found a - // face of an active - // cell that has - // children. that means - // that there are - // hanging nodes here. - - // in any case, faces - // can have at most two - // active fe indices, - // but here the face - // can have only one - // (namely the same as - // that from the cell - // we're sitting on), - // and each of the - // children can have - // only one as - // well. check this - AssertDimension (cell->face(face)->n_active_fe_indices(), 1); - Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) - == true, - ExcInternalError()); - for (unsigned int c=0; cface(face)->n_children(); ++c) - AssertDimension (cell->face(face)->child(c)->n_active_fe_indices(), 1); - - // right now, all that - // is implemented is - // the case that both - // sides use the same - // fe, and not only - // that but also that - // all lines bounding - // this face and the - // children have the - // same fe - for (unsigned int c=0; cface(face)->n_children(); ++c) - { - Assert (cell->face(face)->child(c) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcNotImplemented()); - for (unsigned int e=0; e<4; ++e) + // artificial cells can at + // best neighbor ghost cells, + // but we're not interested + // in these interfaces + if (!cell->is_artificial ()) + for (unsigned int face=0; face::faces_per_cell; ++face) + if (cell->face(face)->has_children()) + { + // first of all, make sure that + // we treat a case which is + // possible, i.e. either no dofs + // on the face at all or no + // anisotropic refinement + if (cell->get_fe().dofs_per_face == 0) + continue; + + Assert(cell->face(face)->refinement_case()==RefinementCase::isotropic_refinement, + ExcNotImplemented()); + + // in any case, faces + // can have at most two + // active fe indices, + // but here the face + // can have only one + // (namely the same as + // that from the cell + // we're sitting on), + // and each of the + // children can have + // only one as + // well. check this + AssertDimension (cell->face(face)->n_active_fe_indices(), 1); + Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) + == true, + ExcInternalError()); + for (unsigned int c=0; cface(face)->n_children(); ++c) + AssertDimension (cell->face(face)->child(c)->n_active_fe_indices(), 1); + + // right now, all that + // is implemented is + // the case that both + // sides use the same + // fe, and not only + // that but also that + // all lines bounding + // this face and the + // children have the + // same fe + for (unsigned int c=0; cface(face)->n_children(); ++c) + if (!cell->neighbor_child_on_subface(face,c)->is_artificial()) { - Assert (cell->face(face)->child(c)->line(e) - ->n_active_fe_indices() == 1, - ExcNotImplemented()); - Assert (cell->face(face)->child(c)->line(e) + Assert (cell->face(face)->child(c) ->fe_index_is_active(cell->active_fe_index()) == true, ExcNotImplemented()); + for (unsigned int e=0; e<4; ++e) + { + Assert (cell->face(face)->child(c)->line(e) + ->n_active_fe_indices() == 1, + ExcNotImplemented()); + Assert (cell->face(face)->child(c)->line(e) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcNotImplemented()); + } } - } - for (unsigned int e=0; e<4; ++e) - { - Assert (cell->face(face)->line(e) - ->n_active_fe_indices() == 1, - ExcNotImplemented()); - Assert (cell->face(face)->line(e) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcNotImplemented()); - } - - // ok, start up the work - const FiniteElement &fe = cell->get_fe(); - const unsigned int fe_index = cell->active_fe_index(); - - const unsigned int - n_dofs_on_mother = (4*fe.dofs_per_vertex+ - 4*fe.dofs_per_line+ - fe.dofs_per_quad), - n_dofs_on_children = (5*fe.dofs_per_vertex+ - 12*fe.dofs_per_line+ - 4*fe.dofs_per_quad); - //TODO[TL]: think about this and the following in case of anisotropic refinement - - dofs_on_mother.resize (n_dofs_on_mother); - dofs_on_children.resize (n_dofs_on_children); - - Assert(n_dofs_on_mother == fe.constraints().n(), - ExcDimensionMismatch(n_dofs_on_mother, - fe.constraints().n())); - Assert(n_dofs_on_children == fe.constraints().m(), - ExcDimensionMismatch(n_dofs_on_children, - fe.constraints().m())); - - const typename DH::face_iterator this_face = cell->face(face); - - // fill the dofs indices. Use same - // enumeration scheme as in - // @p{FiniteElement::constraints()} - unsigned int next_index = 0; - for (unsigned int vertex=0; vertex<4; ++vertex) - for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) - dofs_on_mother[next_index++] = this_face->vertex_dof_index(vertex,dof, - fe_index); - for (unsigned int line=0; line<4; ++line) - for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) - dofs_on_mother[next_index++] - = this_face->line(line)->dof_index(dof, fe_index); - for (unsigned int dof=0; dof!=fe.dofs_per_quad; ++dof) - dofs_on_mother[next_index++] = this_face->dof_index(dof, fe_index); - AssertDimension (next_index, dofs_on_mother.size()); - - next_index = 0; - - // assert some consistency - // assumptions - //TODO[TL]: think about this in case of anisotropic refinement - Assert (dof_handler.get_tria().get_anisotropic_refinement_flag() || - ((this_face->child(0)->vertex_index(3) == - this_face->child(1)->vertex_index(2)) && - (this_face->child(0)->vertex_index(3) == - this_face->child(2)->vertex_index(1)) && - (this_face->child(0)->vertex_index(3) == - this_face->child(3)->vertex_index(0))), - ExcInternalError()); - for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) - dofs_on_children[next_index++] - = this_face->child(0)->vertex_dof_index(3,dof); + for (unsigned int e=0; e<4; ++e) + { + Assert (cell->face(face)->line(e) + ->n_active_fe_indices() == 1, + ExcNotImplemented()); + Assert (cell->face(face)->line(e) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcNotImplemented()); + } - // dof numbers on the centers of - // the lines bounding this face - for (unsigned int line=0; line<4; ++line) + // ok, start up the work + const FiniteElement &fe = cell->get_fe(); + const unsigned int fe_index = cell->active_fe_index(); + + const unsigned int + n_dofs_on_mother = (4*fe.dofs_per_vertex+ + 4*fe.dofs_per_line+ + fe.dofs_per_quad), + n_dofs_on_children = (5*fe.dofs_per_vertex+ + 12*fe.dofs_per_line+ + 4*fe.dofs_per_quad); + //TODO[TL]: think about this and the following in case of anisotropic refinement + + dofs_on_mother.resize (n_dofs_on_mother); + dofs_on_children.resize (n_dofs_on_children); + + Assert(n_dofs_on_mother == fe.constraints().n(), + ExcDimensionMismatch(n_dofs_on_mother, + fe.constraints().n())); + Assert(n_dofs_on_children == fe.constraints().m(), + ExcDimensionMismatch(n_dofs_on_children, + fe.constraints().m())); + + const typename DH::face_iterator this_face = cell->face(face); + + // fill the dofs indices. Use same + // enumeration scheme as in + // @p{FiniteElement::constraints()} + unsigned int next_index = 0; + for (unsigned int vertex=0; vertex<4; ++vertex) + for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) + dofs_on_mother[next_index++] = this_face->vertex_dof_index(vertex,dof, + fe_index); + for (unsigned int line=0; line<4; ++line) + for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) + dofs_on_mother[next_index++] + = this_face->line(line)->dof_index(dof, fe_index); + for (unsigned int dof=0; dof!=fe.dofs_per_quad; ++dof) + dofs_on_mother[next_index++] = this_face->dof_index(dof, fe_index); + AssertDimension (next_index, dofs_on_mother.size()); + + next_index = 0; + + // assert some consistency + // assumptions + //TODO[TL]: think about this in case of anisotropic refinement + Assert (dof_handler.get_tria().get_anisotropic_refinement_flag() || + ((this_face->child(0)->vertex_index(3) == + this_face->child(1)->vertex_index(2)) && + (this_face->child(0)->vertex_index(3) == + this_face->child(2)->vertex_index(1)) && + (this_face->child(0)->vertex_index(3) == + this_face->child(3)->vertex_index(0))), + ExcInternalError()); for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) dofs_on_children[next_index++] - = this_face->line(line)->child(0)->vertex_dof_index(1,dof, fe_index); - - // next the dofs on the lines interior - // to the face; the order of these - // lines is laid down in the - // FiniteElement class documentation - for (unsigned int dof=0; dofchild(0)->line(1)->dof_index(dof, fe_index); - for (unsigned int dof=0; dofchild(2)->line(1)->dof_index(dof, fe_index); - for (unsigned int dof=0; dofchild(0)->line(3)->dof_index(dof, fe_index); - for (unsigned int dof=0; dofchild(1)->line(3)->dof_index(dof, fe_index); + = this_face->child(0)->vertex_dof_index(3,dof); - // dofs on the bordering lines - for (unsigned int line=0; line<4; ++line) - for (unsigned int child=0; child<2; ++child) - for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) + // dof numbers on the centers of + // the lines bounding this face + for (unsigned int line=0; line<4; ++line) + for (unsigned int dof=0; dof!=fe.dofs_per_vertex; ++dof) dofs_on_children[next_index++] - = this_face->line(line)->child(child)->dof_index(dof, fe_index); + = this_face->line(line)->child(0)->vertex_dof_index(1,dof, fe_index); - // finally, for the dofs interior - // to the four child faces - for (unsigned int child=0; child<4; ++child) - for (unsigned int dof=0; dof!=fe.dofs_per_quad; ++dof) + // next the dofs on the lines interior + // to the face; the order of these + // lines is laid down in the + // FiniteElement class documentation + for (unsigned int dof=0; dofchild(child)->dof_index(dof, fe_index); - AssertDimension (next_index, dofs_on_children.size()); + = this_face->child(0)->line(1)->dof_index(dof, fe_index); + for (unsigned int dof=0; dofchild(2)->line(1)->dof_index(dof, fe_index); + for (unsigned int dof=0; dofchild(0)->line(3)->dof_index(dof, fe_index); + for (unsigned int dof=0; dofchild(1)->line(3)->dof_index(dof, fe_index); + + // dofs on the bordering lines + for (unsigned int line=0; line<4; ++line) + for (unsigned int child=0; child<2; ++child) + for (unsigned int dof=0; dof!=fe.dofs_per_line; ++dof) + dofs_on_children[next_index++] + = this_face->line(line)->child(child)->dof_index(dof, fe_index); + + // finally, for the dofs interior + // to the four child faces + for (unsigned int child=0; child<4; ++child) + for (unsigned int dof=0; dof!=fe.dofs_per_quad; ++dof) + dofs_on_children[next_index++] + = this_face->child(child)->dof_index(dof, fe_index); + AssertDimension (next_index, dofs_on_children.size()); - // for each row in the constraint - // matrix for this line: - for (unsigned int row=0; row!=dofs_on_children.size(); ++row) - { - constraints.add_line (dofs_on_children[row]); - for (unsigned int i=0; i!=dofs_on_mother.size(); ++i) - constraints.add_entry (dofs_on_children[row], - dofs_on_mother[i], - fe.constraints()(row,i)); + // for each row in the constraint + // matrix for this line: + for (unsigned int row=0; row!=dofs_on_children.size(); ++row) + { + constraints.add_line (dofs_on_children[row]); + for (unsigned int i=0; i!=dofs_on_mother.size(); ++i) + constraints.add_entry (dofs_on_children[row], + dofs_on_mother[i], + fe.constraints()(row,i)); - constraints.set_inhomogeneity(dofs_on_children[row], 0.); - } - } - else - { - // this face has no - // children, but it - // could still be that - // it is shared by two - // cells that use a - // different fe index - Assert (cell->face(face)->n_active_fe_indices() == 1, - ExcNotImplemented()); - Assert (cell->face(face) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcInternalError()); - } + constraints.set_inhomogeneity(dofs_on_children[row], 0.); + } + } + else + { + // this face has no + // children, but it + // could still be + // that it is shared + // by two cells that + // use a different fe + // index. check a + // couple of things, + // but ignore the + // case that the + // neighbor is an + // artificial cell + if (!cell->at_boundary(face) && + !cell->neighbor(face)->is_artificial()) + { + Assert (cell->face(face)->n_active_fe_indices() == 1, + ExcNotImplemented()); + Assert (cell->face(face) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcInternalError()); + } + } } #endif @@ -2418,638 +2457,742 @@ namespace internal typename DH::active_cell_iterator cell = dof_handler.begin_active(), endc = dof_handler.end(); for (; cell!=endc; ++cell) - for (unsigned int face=0; face::faces_per_cell; ++face) - if (cell->face(face)->has_children()) - { - // first of all, make sure that - // we treat a case which is - // possible, i.e. either no dofs - // on the face at all or no - // anisotropic refinement - if (cell->get_fe().dofs_per_face == 0) - continue; - - Assert(cell->face(face)->refinement_case()==RefinementCase::isotropic_refinement, - ExcNotImplemented()); - - // check to see if the child elements - // have no dofs on the - // shared face. if none of them do, we - // we can simply continue to the next face. - // if only some of them have dofs, while - // others do not, then we don't know - // what to do and throw an exception. - bool any_are_zero = false; - bool all_are_zero = true; - - for (unsigned int c=0; cface(face)->n_children(); ++c) - { - if(cell->neighbor_child_on_subface (face, c)->get_fe().dofs_per_face == 0) - any_are_zero = true; - else - all_are_zero = false; - } - - if(all_are_zero) - continue; - - Assert( all_are_zero || !any_are_zero, ExcNotImplemented() ); - - // so now we've found a - // face of an active - // cell that has - // children. that means - // that there are - // hanging nodes here. - - // in any case, faces - // can have at most two - // sets of active fe - // indices, but here - // the face can have - // only one (namely the - // same as that from - // the cell we're - // sitting on), and - // each of the children - // can have only one as - // well. check this - Assert (cell->face(face)->n_active_fe_indices() == 1, - ExcInternalError()); - Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) - == true, - ExcInternalError()); - for (unsigned int c=0; cface(face)->n_children(); ++c) - Assert (cell->face(face)->child(c)->n_active_fe_indices() == 1, - ExcInternalError()); + // artificial cells can at + // best neighbor ghost cells, + // but we're not interested + // in these interfaces + if (!cell->is_artificial ()) + for (unsigned int face=0; face::faces_per_cell; ++face) + if (cell->face(face)->has_children()) + { + // first of all, make sure that + // we treat a case which is + // possible, i.e. either no dofs + // on the face at all or no + // anisotropic refinement + if (cell->get_fe().dofs_per_face == 0) + continue; + + Assert(cell->face(face)->refinement_case()==RefinementCase::isotropic_refinement, + ExcNotImplemented()); + + // check to see if the child elements + // have no dofs on the + // shared face. if none of them do, we + // we can simply continue to the next face. + // if only some of them have dofs, while + // others do not, then we don't know + // what to do and throw an exception. + // + // ignore all + // interfaces with + // artificial cells + bool any_are_zero = false; + bool all_are_zero = true; + + for (unsigned int c=0; cface(face)->n_children(); ++c) + if (!cell->neighbor_child_on_subface(face, c)->is_artificial()) + { + if (cell->neighbor_child_on_subface(face, c)->get_fe().dofs_per_face == 0) + any_are_zero = true; + else + all_are_zero = false; + } - // first find out - // whether we can - // constrain each of - // the subfaces to the - // mother face. in the - // lingo of the hp - // paper, this would be - // the simple - // case. note that we - // can short-circuit - // this decision if the - // dof_handler doesn't - // support hp at all - FiniteElementDomination::Domination - mother_face_dominates = FiniteElementDomination::either_element_can_dominate; - - if (DoFHandlerSupportsDifferentFEs::value == true) - for (unsigned int c=0; cface(face)->number_of_children(); ++c) - mother_face_dominates = mother_face_dominates & - (cell->get_fe().compare_for_face_domination - (cell->neighbor_child_on_subface (face, c)->get_fe())); - - switch (mother_face_dominates) - { - case FiniteElementDomination::this_element_dominates: - case FiniteElementDomination::either_element_can_dominate: + if(all_are_zero) + continue; + + Assert( all_are_zero || !any_are_zero, ExcNotImplemented() ); + + // so now we've found a + // face of an active + // cell that has + // children. that means + // that there are + // hanging nodes here. + + // in any case, faces + // can have at most two + // sets of active fe + // indices, but here + // the face can have + // only one (namely the + // same as that from + // the cell we're + // sitting on), and + // each of the children + // can have only one as + // well. check this + Assert (cell->face(face)->n_active_fe_indices() == 1, + ExcInternalError()); + Assert (cell->face(face)->fe_index_is_active(cell->active_fe_index()) + == true, + ExcInternalError()); + for (unsigned int c=0; cface(face)->n_children(); ++c) + Assert (cell->face(face)->child(c)->n_active_fe_indices() == 1, + ExcInternalError()); + + // first find out + // whether we can + // constrain each of + // the subfaces to the + // mother face. in the + // lingo of the hp + // paper, this would be + // the simple + // case. note that we + // can short-circuit + // this decision if the + // dof_handler doesn't + // support hp at all + // + // ignore all + // interfaces with + // artificial cells + FiniteElementDomination::Domination + mother_face_dominates = FiniteElementDomination::either_element_can_dominate; + + if (DoFHandlerSupportsDifferentFEs::value == true) + for (unsigned int c=0; cface(face)->number_of_children(); ++c) + if (!cell->neighbor_child_on_subface (face, c)->is_artificial()) + mother_face_dominates = mother_face_dominates & + (cell->get_fe().compare_for_face_domination + (cell->neighbor_child_on_subface (face, c)->get_fe())); + + switch (mother_face_dominates) { - // Case 1 (the - // simple case - // and the only - // case that can - // happen for - // non-hp - // DoFHandlers): - // The coarse - // element - // dominates the - // elements on - // the subfaces - // (or they are - // all the same) - master_dofs.resize (cell->get_fe().dofs_per_face); - - cell->face(face)->get_dof_indices (master_dofs, - cell->active_fe_index ()); - - // Now create constraint matrix for - // the subfaces and assemble it. - for (unsigned int c=0; cface(face)->n_children(); ++c) - { - const typename DH::active_face_iterator - subface = cell->face(face)->child(c); - - Assert (subface->n_active_fe_indices() == 1, - ExcInternalError()); - - const unsigned int - subface_fe_index = subface->nth_active_fe_index(0); - - // Same procedure as for the - // mother cell. Extract the face - // DoFs from the cell DoFs. - slave_dofs.resize (subface->get_fe(subface_fe_index) - .dofs_per_face); - subface->get_dof_indices (slave_dofs, subface_fe_index); - - // Now create the - // element constraint - // for this subface. - // - // As a side remark, - // one may wonder the - // following: - // neighbor_child is - // clearly computed - // correctly, - // i.e. taking into - // account - // face_orientation - // (just look at the - // implementation of - // that - // function). however, - // we don't care about - // this here, when we - // ask for - // subface_interpolation - // on subface c. the - // question rather is: - // do we have to - // translate 'c' here - // as well? - // - // the answer is in - // fact 'no'. if one - // does that, results - // are wrong: - // constraints are - // added twice for the - // same pair of nodes - // but with differing - // weights. in - // addition, one can - // look at the - // deal.II/project_*_03 - // tests that look at - // exactly this case: - // there, we have a - // mesh with at least - // one - // face_orientation==false - // and hanging nodes, - // and the results of - // those tests show - // that the result of - // projection verifies - // the approximation - // properties of a - // finite element onto - // that mesh - ensure_existence_of_subface_matrix - (cell->get_fe(), - subface->get_fe(subface_fe_index), - c, - subface_interpolation_matrices - [cell->active_fe_index()][subface_fe_index][c]); - - // Add constraints to global constraint - // matrix. + case FiniteElementDomination::this_element_dominates: + case FiniteElementDomination::either_element_can_dominate: + { + // Case 1 (the + // simple case + // and the only + // case that can + // happen for + // non-hp + // DoFHandlers): + // The coarse + // element + // dominates the + // elements on + // the subfaces + // (or they are + // all the same) + // + // so we are + // going to + // constrain + // the DoFs on + // the face + // children + // against the + // DoFs on the + // face itself + master_dofs.resize (cell->get_fe().dofs_per_face); + + cell->face(face)->get_dof_indices (master_dofs, + cell->active_fe_index ()); + + // Now create + // constraint matrix + // for the subfaces and + // assemble it. ignore + // all interfaces with + // artificial cells + // because we can only + // get to such + // interfaces if the + // current cell is a + // ghost cell. also + // ignore the interface + // if the neighboring + // cell is a ghost cell + // in 2d: what we would + // compute here are the + // constraints on the + // ghost cell's DoFs, + // but we are not + // interested in those: + // we only want + // constraints on + // *locally active* + // DoFs, not on + // *locally relevant* + // DoFs. However, in 3d + // we must still + // compute those + // constraints because + // it might happen that + // a constraint is + // related to an edge + // where the hanging + // node is only + // detected if we also + // look between ghosts + for (unsigned int c=0; cface(face)->n_children(); ++c) + { + if (cell->neighbor_child_on_subface (face, c)->is_artificial() + || + (dim == 2 && cell->neighbor_child_on_subface (face, c)->is_ghost())) + continue; + + const typename DH::active_face_iterator + subface = cell->face(face)->child(c); + + Assert (subface->n_active_fe_indices() == 1, + ExcInternalError()); + + const unsigned int + subface_fe_index = subface->nth_active_fe_index(0); + + // Same procedure as for the + // mother cell. Extract the face + // DoFs from the cell DoFs. + slave_dofs.resize (subface->get_fe(subface_fe_index) + .dofs_per_face); + subface->get_dof_indices (slave_dofs, subface_fe_index); + + for (unsigned int i=0; iget_fe(), + subface->get_fe(subface_fe_index), + c, + subface_interpolation_matrices + [cell->active_fe_index()][subface_fe_index][c]); + + // Add constraints to global constraint + // matrix. #ifdef WOLFGANG - std::cout << "Constraints for cell=" << cell - << ", face=" << face - << ", subface=" << c - << std::endl; + std::cout << "Constraints for cell=" << cell + << ", face=" << face + << ", subface=" << c + << std::endl; #endif - filter_constraints (master_dofs, - slave_dofs, - *(subface_interpolation_matrices - [cell->active_fe_index()][subface_fe_index][c]), - constraints); - } + filter_constraints (master_dofs, + slave_dofs, + *(subface_interpolation_matrices + [cell->active_fe_index()][subface_fe_index][c]), + constraints); + } - break; - } + break; + } - case FiniteElementDomination::other_element_dominates: - case FiniteElementDomination::neither_element_dominates: - { - // Case 2 (the "complex" - // case): at least one - // (the neither_... case) - // of the finer elements - // or all of them (the - // other_... case) is - // dominating. See the hp - // paper for a way how to - // deal with this - // situation - // - // since this is - // something that - // can only - // happen for hp - // dof handlers, - // add a check - // here... - Assert (DoFHandlerSupportsDifferentFEs::value == true, - ExcInternalError()); - - // we first have - // to find the - // finite element - // that is able - // to generate a - // space that all - // the other ones - // can be - // constrained to - const unsigned int dominating_fe_index - = get_most_dominating_subface_fe_index (cell->face(face)); - - const FiniteElement &dominating_fe - = dof_handler.get_fe()[dominating_fe_index]; - - // check also - // that it is - // able to - // constrain the - // mother - // face. it - // should be, or - // we wouldn't - // have gotten - // into the - // branch for the - // 'complex' case - Assert ((dominating_fe.compare_for_face_domination - (cell->face(face)->get_fe(cell->face(face)->nth_active_fe_index(0))) - == FiniteElementDomination::this_element_dominates) - || - (dominating_fe.compare_for_face_domination - (cell->face(face)->get_fe(cell->face(face)->nth_active_fe_index(0))) - == FiniteElementDomination::either_element_can_dominate), - ExcInternalError()); + case FiniteElementDomination::other_element_dominates: + case FiniteElementDomination::neither_element_dominates: + { + // Case 2 (the "complex" + // case): at least one + // (the neither_... case) + // of the finer elements + // or all of them (the + // other_... case) is + // dominating. See the hp + // paper for a way how to + // deal with this + // situation + // + // since this is + // something that + // can only + // happen for hp + // dof handlers, + // add a check + // here... + Assert (DoFHandlerSupportsDifferentFEs::value == true, + ExcInternalError()); + // we first have + // to find the + // finite element + // that is able + // to generate a + // space that all + // the other ones + // can be + // constrained to + const unsigned int dominating_fe_index + = get_most_dominating_subface_fe_index (cell->face(face)); + + const FiniteElement &dominating_fe + = dof_handler.get_fe()[dominating_fe_index]; + + // check also + // that it is + // able to + // constrain the + // mother + // face. it + // should be, or + // we wouldn't + // have gotten + // into the + // branch for the + // 'complex' case + Assert ((dominating_fe.compare_for_face_domination + (cell->face(face)->get_fe(cell->face(face)->nth_active_fe_index(0))) + == FiniteElementDomination::this_element_dominates) + || + (dominating_fe.compare_for_face_domination + (cell->face(face)->get_fe(cell->face(face)->nth_active_fe_index(0))) + == FiniteElementDomination::either_element_can_dominate), + ExcInternalError()); - // first get the - // interpolation matrix - // from the mother to the - // virtual dofs - Assert (dominating_fe.dofs_per_face <= - cell->get_fe().dofs_per_face, - ExcInternalError()); - ensure_existence_of_face_matrix - (dominating_fe, - cell->get_fe(), - face_interpolation_matrices - [dominating_fe_index][cell->active_fe_index()]); - - // split this matrix into - // master and slave - // components. invert the - // master component - ensure_existence_of_master_dof_mask - (cell->get_fe(), - dominating_fe, - (*face_interpolation_matrices - [dominating_fe_index] - [cell->active_fe_index()]), - master_dof_masks - [dominating_fe_index] - [cell->active_fe_index()]); - - ensure_existence_of_split_face_matrix - (*face_interpolation_matrices - [dominating_fe_index][cell->active_fe_index()], - (*master_dof_masks - [dominating_fe_index][cell->active_fe_index()]), - split_face_interpolation_matrices - [dominating_fe_index][cell->active_fe_index()]); - - const FullMatrix &restrict_mother_to_virtual_master_inv - = (split_face_interpolation_matrices - [dominating_fe_index][cell->active_fe_index()]->first); - - const FullMatrix &restrict_mother_to_virtual_slave - = (split_face_interpolation_matrices - [dominating_fe_index][cell->active_fe_index()]->second); - - // now compute - // the constraint - // matrix as the - // product - // between the - // inverse matrix - // and the slave - // part - constraint_matrix.reinit (cell->get_fe().dofs_per_face - - dominating_fe.dofs_per_face, - dominating_fe.dofs_per_face); - restrict_mother_to_virtual_slave - .mmult (constraint_matrix, - restrict_mother_to_virtual_master_inv); - - // then figure - // out the global - // numbers of - // master and - // slave dofs and - // apply - // constraints - scratch_dofs.resize (cell->get_fe().dofs_per_face); - cell->face(face)->get_dof_indices (scratch_dofs, - cell->active_fe_index ()); - - // split dofs into master - // and slave components - master_dofs.clear (); - slave_dofs.clear (); - for (unsigned int i=0; iget_fe().dofs_per_face; ++i) - if ((*master_dof_masks - [dominating_fe_index][cell->active_fe_index()])[i] == true) - master_dofs.push_back (scratch_dofs[i]); - else - slave_dofs.push_back (scratch_dofs[i]); + // first get the + // interpolation matrix + // from the mother to the + // virtual dofs + Assert (dominating_fe.dofs_per_face <= + cell->get_fe().dofs_per_face, + ExcInternalError()); - AssertDimension (master_dofs.size(), dominating_fe.dofs_per_face); - AssertDimension (slave_dofs.size(), - cell->get_fe().dofs_per_face - dominating_fe.dofs_per_face); + ensure_existence_of_face_matrix + (dominating_fe, + cell->get_fe(), + face_interpolation_matrices + [dominating_fe_index][cell->active_fe_index()]); + + // split this matrix into + // master and slave + // components. invert the + // master component + ensure_existence_of_master_dof_mask + (cell->get_fe(), + dominating_fe, + (*face_interpolation_matrices + [dominating_fe_index] + [cell->active_fe_index()]), + master_dof_masks + [dominating_fe_index] + [cell->active_fe_index()]); + + ensure_existence_of_split_face_matrix + (*face_interpolation_matrices + [dominating_fe_index][cell->active_fe_index()], + (*master_dof_masks + [dominating_fe_index][cell->active_fe_index()]), + split_face_interpolation_matrices + [dominating_fe_index][cell->active_fe_index()]); + + const FullMatrix &restrict_mother_to_virtual_master_inv + = (split_face_interpolation_matrices + [dominating_fe_index][cell->active_fe_index()]->first); + + const FullMatrix &restrict_mother_to_virtual_slave + = (split_face_interpolation_matrices + [dominating_fe_index][cell->active_fe_index()]->second); + + // now compute + // the constraint + // matrix as the + // product + // between the + // inverse matrix + // and the slave + // part + constraint_matrix.reinit (cell->get_fe().dofs_per_face - + dominating_fe.dofs_per_face, + dominating_fe.dofs_per_face); + restrict_mother_to_virtual_slave + .mmult (constraint_matrix, + restrict_mother_to_virtual_master_inv); + + // then figure + // out the global + // numbers of + // master and + // slave dofs and + // apply + // constraints + scratch_dofs.resize (cell->get_fe().dofs_per_face); + cell->face(face)->get_dof_indices (scratch_dofs, + cell->active_fe_index ()); + + // split dofs into master + // and slave components + master_dofs.clear (); + slave_dofs.clear (); + for (unsigned int i=0; iget_fe().dofs_per_face; ++i) + if ((*master_dof_masks + [dominating_fe_index][cell->active_fe_index()])[i] == true) + master_dofs.push_back (scratch_dofs[i]); + else + slave_dofs.push_back (scratch_dofs[i]); + + AssertDimension (master_dofs.size(), dominating_fe.dofs_per_face); + AssertDimension (slave_dofs.size(), + cell->get_fe().dofs_per_face - dominating_fe.dofs_per_face); #ifdef WOLFGANG - std::cout << "Constraints for cell=" << cell - << ", face=" << face - << " (complicated case, mother)" - << std::endl; + std::cout << "Constraints for cell=" << cell + << ", face=" << face + << " (complicated case, mother)" + << std::endl; #endif - filter_constraints (master_dofs, - slave_dofs, - constraint_matrix, - constraints); + filter_constraints (master_dofs, + slave_dofs, + constraint_matrix, + constraints); + + + + // next we have + // to deal with + // the + // subfaces. do + // as discussed + // in the hp + // paper + for (unsigned int sf=0; + sfface(face)->n_children(); ++sf) + { + // ignore + // interfaces + // with + // artificial + // cells as + // well as + // interfaces + // between + // ghost + // cells in 2d + if (cell->neighbor_child_on_subface (face, sf)->is_artificial() + || + (dim==2 && cell->is_ghost() + && + cell->neighbor_child_on_subface (face, sf)->is_ghost())) + continue; + + Assert (cell->face(face)->child(sf) + ->n_active_fe_indices() == 1, + ExcInternalError()); + + const unsigned int subface_fe_index + = cell->face(face)->child(sf)->nth_active_fe_index(0); + const FiniteElement &subface_fe + = dof_handler.get_fe()[subface_fe_index]; + + // first get the + // interpolation + // matrix from the + // subface to the + // virtual dofs + Assert (dominating_fe.dofs_per_face <= + subface_fe.dofs_per_face, + ExcInternalError()); + ensure_existence_of_subface_matrix + (dominating_fe, + subface_fe, + sf, + subface_interpolation_matrices + [dominating_fe_index][subface_fe_index][sf]); + const FullMatrix &restrict_subface_to_virtual + = *(subface_interpolation_matrices + [dominating_fe_index][subface_fe_index][sf]); + constraint_matrix.reinit (subface_fe.dofs_per_face, + dominating_fe.dofs_per_face); - // next we have to - // deal with the - // subfaces. do as - // discussed in the - // paper - for (unsigned int sf=0; - sfface(face)->n_children(); ++sf) - { - Assert (cell->face(face)->child(sf) - ->n_active_fe_indices() == 1, - ExcInternalError()); - - const unsigned int subface_fe_index - = cell->face(face)->child(sf)->nth_active_fe_index(0); - const FiniteElement &subface_fe - = dof_handler.get_fe()[subface_fe_index]; - - // first get the - // interpolation - // matrix from the - // subface to the - // virtual dofs - Assert (dominating_fe.dofs_per_face <= - subface_fe.dofs_per_face, - ExcInternalError()); - ensure_existence_of_subface_matrix - (dominating_fe, - subface_fe, - sf, - subface_interpolation_matrices - [dominating_fe_index][subface_fe_index][sf]); - - const FullMatrix &restrict_subface_to_virtual - = *(subface_interpolation_matrices - [dominating_fe_index][subface_fe_index][sf]); - - constraint_matrix.reinit (subface_fe.dofs_per_face, - dominating_fe.dofs_per_face); - - restrict_subface_to_virtual - .mmult (constraint_matrix, - restrict_mother_to_virtual_master_inv); - - slave_dofs.resize (subface_fe.dofs_per_face); - cell->face(face)->child(sf)->get_dof_indices (slave_dofs, - subface_fe_index); + restrict_subface_to_virtual + .mmult (constraint_matrix, + restrict_mother_to_virtual_master_inv); + + slave_dofs.resize (subface_fe.dofs_per_face); + cell->face(face)->child(sf)->get_dof_indices (slave_dofs, + subface_fe_index); #ifdef WOLFGANG - std::cout << "Constraints for cell=" << cell - << ", face=" << face - << ", subface=" << sf - << " (complicated case, children)" - << std::endl; + std::cout << "Constraints for cell=" << cell + << ", face=" << face + << ", subface=" << sf + << " (complicated case, children)" + << std::endl; #endif - filter_constraints (master_dofs, - slave_dofs, - constraint_matrix, - constraints); - } + filter_constraints (master_dofs, + slave_dofs, + constraint_matrix, + constraints); + } - break; + break; + } + + default: + // we shouldn't get here + Assert (false, ExcInternalError()); } + } + else + { + // this face has no + // children, but it + // could still be that + // it is shared by two + // cells that use a + // different fe index + Assert (cell->face(face) + ->fe_index_is_active(cell->active_fe_index()) == true, + ExcInternalError()); - default: - // we shouldn't get here - Assert (false, ExcInternalError()); - } - } - else - { - // this face has no - // children, but it - // could still be that - // it is shared by two - // cells that use a - // different fe index - Assert (cell->face(face) - ->fe_index_is_active(cell->active_fe_index()) == true, - ExcInternalError()); - - // Only if there is - // a neighbor with - // a different - // active_fe_index - // and the same h-level, - // some action has - // to be taken. - if ((DoFHandlerSupportsDifferentFEs::value == true) - && - !cell->face(face)->at_boundary () - && - (cell->neighbor(face)->active_fe_index () != - cell->active_fe_index ()) - && - (!cell->face(face)->has_children() && - !cell->neighbor_is_coarser(face) )) - { - const typename DH::cell_iterator neighbor = cell->neighbor (face); + // see if there is a + // neighbor that is + // an artificial + // cell. in that + // case, we're not + // interested in this + // interface. we test + // this case first + // since artificial + // cells may not have + // an active_fe_index + // set, etc + if (!cell->at_boundary(face) + && + cell->neighbor(face)->is_artificial()) + continue; + + // Only if there is + // a neighbor with + // a different + // active_fe_index + // and the same h-level, + // some action has + // to be taken. + if ((DoFHandlerSupportsDifferentFEs::value == true) + && + !cell->face(face)->at_boundary () + && + (cell->neighbor(face)->active_fe_index () != + cell->active_fe_index ()) + && + (!cell->face(face)->has_children() && + !cell->neighbor_is_coarser(face) )) + { + const typename DH::cell_iterator neighbor = cell->neighbor (face); - // see which side of the - // face we have to - // constrain - switch (cell->get_fe().compare_for_face_domination (neighbor->get_fe ())) - { - case FiniteElementDomination::this_element_dominates: + // see which side of the + // face we have to + // constrain + switch (cell->get_fe().compare_for_face_domination (neighbor->get_fe ())) { - // Get DoFs on - // dominating and - // dominated side of - // the face - master_dofs.resize (cell->get_fe().dofs_per_face); - cell->face(face)->get_dof_indices (master_dofs, - cell->active_fe_index ()); - - slave_dofs.resize (neighbor->get_fe().dofs_per_face); - cell->face(face)->get_dof_indices (slave_dofs, - neighbor->active_fe_index ()); - - // break if the n_master_dofs == 0, - // because we are attempting to - // constrain to an element that has - // has no face dofs - if(master_dofs.size() == 0) break; - - // make sure - // the element - // constraints - // for this - // face are - // available - ensure_existence_of_face_matrix - (cell->get_fe(), - neighbor->get_fe(), - face_interpolation_matrices - [cell->active_fe_index()][neighbor->active_fe_index()]); - - // Add constraints to global constraint - // matrix. + case FiniteElementDomination::this_element_dominates: + { + // Get DoFs on + // dominating and + // dominated side of + // the face + master_dofs.resize (cell->get_fe().dofs_per_face); + cell->face(face)->get_dof_indices (master_dofs, + cell->active_fe_index ()); + + slave_dofs.resize (neighbor->get_fe().dofs_per_face); + cell->face(face)->get_dof_indices (slave_dofs, + neighbor->active_fe_index ()); + + // break if the n_master_dofs == 0, + // because we are attempting to + // constrain to an element that has + // has no face dofs + if(master_dofs.size() == 0) break; + + // make sure + // the element + // constraints + // for this + // face are + // available + ensure_existence_of_face_matrix + (cell->get_fe(), + neighbor->get_fe(), + face_interpolation_matrices + [cell->active_fe_index()][neighbor->active_fe_index()]); + + // Add constraints to global constraint + // matrix. #ifdef WOLFGANG - std::cout << "p-constraints for cell=" << cell - << ", face=" << face << std::endl; + std::cout << "p-constraints for cell=" << cell + << ", face=" << face << std::endl; #endif - filter_constraints (master_dofs, - slave_dofs, - *(face_interpolation_matrices - [cell->active_fe_index()] - [neighbor->active_fe_index()]), - constraints); + filter_constraints (master_dofs, + slave_dofs, + *(face_interpolation_matrices + [cell->active_fe_index()] + [neighbor->active_fe_index()]), + constraints); + + break; + } - break; - } + case FiniteElementDomination::other_element_dominates: + { + // we don't do anything + // here since we will + // come back to this + // face from the other + // cell, at which time + // we will fall into + // the first case + // clause above + break; + } - case FiniteElementDomination::other_element_dominates: - { - // we don't do anything - // here since we will - // come back to this - // face from the other - // cell, at which time - // we will fall into - // the first case - // clause above - break; - } + case FiniteElementDomination::either_element_can_dominate: + { + // it appears as if + // neither element has + // any constraints on + // its neighbor. this + // may be because + // neither element has + // any DoFs on faces at + // all. or that the two + // elements are + // actually the same, + // although they happen + // to run under + // different fe_indices + // (this is what + // happens in + // hp/hp_hanging_nodes_01 + // for example). + // + // another possibility + // is what happens in + // crash_13. there, we + // have + // FESystem(FE_Q(1),FE_DGQ(0)) + // vs. FESystem(FE_Q(1),FE_DGQ(1)). + // neither of them + // dominates the + // other. the point is + // that it doesn't + // matter, since + // hopefully for this + // case, both sides' + // dofs should have + // been unified. + // + // make sure this is + // actually true. this + // actually only + // matters, of course, + // if either of the two + // finite elements + // actually do have + // dofs on the face + if ((cell->get_fe().dofs_per_face != 0) + || + (cell->neighbor(face)->get_fe().dofs_per_face != 0)) + { + Assert (cell->get_fe().dofs_per_face + == + cell->neighbor(face)->get_fe().dofs_per_face, + ExcNotImplemented()); + + // (ab)use the master + // and slave dofs + // arrays for a + // moment here + master_dofs.resize (cell->get_fe().dofs_per_face); + cell->face(face) + ->get_dof_indices (master_dofs, + cell->active_fe_index ()); + + slave_dofs.resize (cell->neighbor(face)->get_fe().dofs_per_face); + cell->face(face) + ->get_dof_indices (slave_dofs, + cell->neighbor(face)->active_fe_index ()); + + for (unsigned int i=0; iget_fe().dofs_per_face; ++i) + AssertDimension (master_dofs[i], slave_dofs[i]); + } - case FiniteElementDomination::either_element_can_dominate: - { - // it appears as if - // neither element has - // any constraints on - // its neighbor. this - // may be because - // neither element has - // any DoFs on faces at - // all. or that the two - // elements are - // actually the same, - // although they happen - // to run under - // different fe_indices - // (this is what - // happens in - // hp/hp_hanging_nodes_01 - // for example). - // - // another possibility - // is what happens in - // crash_13. there, we - // have - // FESystem(FE_Q(1),FE_DGQ(0)) - // vs. FESystem(FE_Q(1),FE_DGQ(1)). - // neither of them - // dominates the - // other. the point is - // that it doesn't - // matter, since - // hopefully for this - // case, both sides' - // dofs should have - // been unified. - // - // make sure this is - // actually true. this - // actually only - // matters, of course, - // if either of the two - // finite elements - // actually do have - // dofs on the face - if ((cell->get_fe().dofs_per_face != 0) - || - (cell->neighbor(face)->get_fe().dofs_per_face != 0)) - { - Assert (cell->get_fe().dofs_per_face - == - cell->neighbor(face)->get_fe().dofs_per_face, - ExcNotImplemented()); - - // (ab)use the master - // and slave dofs - // arrays for a - // moment here - master_dofs.resize (cell->get_fe().dofs_per_face); - cell->face(face) - ->get_dof_indices (master_dofs, - cell->active_fe_index ()); - - slave_dofs.resize (cell->neighbor(face)->get_fe().dofs_per_face); - cell->face(face) - ->get_dof_indices (slave_dofs, - cell->neighbor(face)->active_fe_index ()); - - for (unsigned int i=0; iget_fe().dofs_per_face; ++i) - AssertDimension (master_dofs[i], slave_dofs[i]); - } + break; + } - break; - } + case FiniteElementDomination::neither_element_dominates: + { + // we don't presently + // know what exactly to + // do here. it isn't quite + // clear what exactly we + // would have to do + // here. sit tight until + // someone trips over the + // following statement + // and see what exactly + // is going on + Assert (false, ExcNotImplemented()); + break; + } - case FiniteElementDomination::neither_element_dominates: - { - // we don't presently - // know what exactly to - // do here. it isn't quite - // clear what exactly we - // would have to do - // here. sit tight until - // someone trips over the - // following statement - // and see what exactly - // is going on - Assert (false, ExcNotImplemented()); - break; + default: + // we shouldn't get + // here + Assert (false, ExcInternalError()); } - - default: - // we shouldn't get - // here - Assert (false, ExcInternalError()); - } - } - } + } + } } } } @@ -3108,8 +3251,9 @@ namespace internal const dealii::hp::FECollection fe_collection (dof.get_fe()); Assert (fe_collection.n_components() < 256, ExcNotImplemented()); - Assert(dofs_by_component.size() == dof.n_dofs(), - ExcDimensionMismatch(dofs_by_component.size(), dof.n_dofs())); + Assert (dofs_by_component.size() == dof.n_locally_owned_dofs(), + ExcDimensionMismatch(dofs_by_component.size(), + dof.n_locally_owned_dofs())); // next set up a table for the // degrees of freedom on each of @@ -3131,7 +3275,8 @@ namespace internal else for (unsigned int i=0; i indices; for (typename DH::active_cell_iterator c=dof.begin_active(); c!=dof.end(); ++ c) - { - const unsigned int fe_index = c->active_fe_index(); - const unsigned int dofs_per_cell = c->get_fe().dofs_per_cell; - indices.resize(dofs_per_cell); - c->get_dof_indices(indices); - for (unsigned int i=0; iis_artificial() && !c->is_ghost()) + { + const unsigned int fe_index = c->active_fe_index(); + const unsigned int dofs_per_cell = c->get_fe().dofs_per_cell; + indices.resize(dofs_per_cell); + c->get_dof_indices(indices); + for (unsigned int i=0; i component_dofs (dof_handler.n_dofs()); + std::vector component_dofs (dof_handler.n_locally_owned_dofs()); std::vector component_mask (dof_handler.get_fe().n_components(), false); component_mask[component] = true; @@ -3306,8 +3454,8 @@ DoFTools::extract_dofs ( ExcDimensionMismatch(component_select.size(), n_components(dof))); } - Assert(selected_dofs.size() == dof.n_dofs(), - ExcDimensionMismatch(selected_dofs.size(), dof.n_dofs())); + Assert(selected_dofs.size() == dof.n_locally_owned_dofs(), + ExcDimensionMismatch(selected_dofs.size(), dof.n_locally_owned_dofs())); // two special cases: no component // is selected, and all components @@ -3316,19 +3464,19 @@ DoFTools::extract_dofs ( if (std::count (component_select.begin(), component_select.end(), true) == 0) { - std::fill_n (selected_dofs.begin(), dof.n_dofs(), false); + std::fill_n (selected_dofs.begin(), dof.n_locally_owned_dofs(), false); return; - }; - if (std::count (component_select.begin(), component_select.end(), true) + } + else if (std::count (component_select.begin(), component_select.end(), true) == static_cast(component_select.size())) { - std::fill_n (selected_dofs.begin(), dof.n_dofs(), true); + std::fill_n (selected_dofs.begin(), dof.n_locally_owned_dofs(), true); return; - }; + } // preset all values by false - std::fill_n (selected_dofs.begin(), dof.n_dofs(), false); + std::fill_n (selected_dofs.begin(), dof.n_locally_owned_dofs(), false); // if we count by blocks, we need to extract // the association of blocks with local dofs, @@ -3337,11 +3485,11 @@ DoFTools::extract_dofs ( // info. Otherwise, we let the function // extract_dofs_by_component function do the // job. - std::vector dofs_by_component (dof.n_dofs()); + std::vector dofs_by_component (dof.n_locally_owned_dofs()); internal::extract_dofs_by_component (dof, component_select, count_by_blocks, dofs_by_component); - for (unsigned int i=0; i &dof_handler, template void -DoFTools::extract_subdomain_dofs (const DH &dof_handler, - const unsigned int subdomain_id, - std::vector &selected_dofs) +DoFTools::extract_subdomain_dofs (const DH &dof_handler, + const types::subdomain_id_t subdomain_id, + std::vector &selected_dofs) { Assert(selected_dofs.size() == dof_handler.n_dofs(), ExcDimensionMismatch(selected_dofs.size(), dof_handler.n_dofs())); @@ -4166,6 +4314,96 @@ DoFTools::extract_subdomain_dofs (const DH &dof_handler, +template +void +DoFTools::extract_locally_owned_dofs (const DH & dof_handler, + IndexSet & dof_set) +{ + // collect all the locally owned dofs + dof_set = dof_handler.locally_owned_dofs(); + dof_set.compress (); +} + + + +template +void +DoFTools::extract_locally_active_dofs (const DH & dof_handler, + IndexSet & dof_set) +{ + // collect all the locally owned dofs + dof_set = dof_handler.locally_owned_dofs(); + + // add the DoF on the adjacent ghost cells + // to the IndexSet, cache them in a + // set. need to check each dof manually + // because we can't be sure that the dof + // range of locally_owned_dofs is really + // contiguous. + std::vector dof_indices; + std::set global_dof_indices; + + typename DH::active_cell_iterator cell = dof_handler.begin_active(), + endc = dof_handler.end(); + for (; cell!=endc; ++cell) + if (!cell->is_ghost() && !cell->is_artificial()) + { + dof_indices.resize(cell->get_fe().dofs_per_cell); + cell->get_dof_indices(dof_indices); + + for (std::vector::iterator it=dof_indices.begin(); + it!=dof_indices.end(); + ++it) + if (!dof_set.is_element(*it)) + global_dof_indices.insert(*it); + } + + dof_set.add_indices(global_dof_indices.begin(), global_dof_indices.end()); + + dof_set.compress(); +} + + + +template +void +DoFTools::extract_locally_relevant_dofs (const DH & dof_handler, + IndexSet & dof_set) +{ + // collect all the locally owned dofs + dof_set = dof_handler.locally_owned_dofs(); + + // add the DoF on the adjacent ghost cells + // to the IndexSet, cache them in a + // set. need to check each dof manually + // because we can't be sure that the dof + // range of locally_owned_dofs is really + // contiguous. + std::vector dof_indices; + std::set global_dof_indices; + + typename DH::active_cell_iterator cell = dof_handler.begin_active(), + endc = dof_handler.end(); + for (; cell!=endc; ++cell) + if (cell->is_ghost()) + { + dof_indices.resize(cell->get_fe().dofs_per_cell); + cell->get_dof_indices(dof_indices); + + for (std::vector::iterator it=dof_indices.begin(); + it!=dof_indices.end(); + ++it) + if (!dof_set.is_element(*it)) + global_dof_indices.insert(*it); + } + + dof_set.add_indices(global_dof_indices.begin(), global_dof_indices.end()); + + dof_set.compress(); +} + + + template void DoFTools::extract_constant_modes (const DH &dof_handler, @@ -4183,7 +4421,7 @@ DoFTools::extract_constant_modes (const DH &dof_handler, if (component_select[i] == true) localized_component[i] = n_components_selected++; - std::vector dofs_by_component (dof_handler.n_dofs()); + std::vector dofs_by_component (dof_handler.n_locally_owned_dofs()); internal::extract_dofs_by_component (dof_handler, component_select, false, dofs_by_component); unsigned int n_selected_dofs = 0; @@ -4201,7 +4439,7 @@ DoFTools::extract_constant_modes (const DH &dof_handler, component_list[d] = component_select[d]; unsigned int counter = 0; - for (unsigned int i=0; i void DoFTools::get_subdomain_association (const DH &dof_handler, - std::vector &subdomain_association) + std::vector &subdomain_association) { + // if the Triangulation is distributed, the + // only thing we can usefully ask is for + // its locally owned subdomain + Assert ((dynamic_cast*> + (&dof_handler.get_tria()) == 0), + ExcMessage ("For parallel::distributed::Triangulation objects and " + "associated DoF handler objects, asking for any subdomain other " + "than the locally owned one does not make sense.")); + Assert(subdomain_association.size() == dof_handler.n_dofs(), ExcDimensionMismatch(subdomain_association.size(), dof_handler.n_dofs())); // preset all values by an invalid value std::fill_n (subdomain_association.begin(), dof_handler.n_dofs(), - numbers::invalid_unsigned_int); + types::invalid_subdomain_id); std::vector local_dof_indices; local_dof_indices.reserve (max_dofs_per_cell(dof_handler)); @@ -4260,7 +4508,11 @@ DoFTools::get_subdomain_association (const DH &dof_handler, endc = dof_handler.end(); for (; cell!=endc; ++cell) { - const unsigned int subdomain_id = cell->subdomain_id(); + Assert (cell->is_artificial() == false, + ExcMessage ("You can't call this function for meshes that " + "have artificial cells.")); + + const types::subdomain_id_t subdomain_id = cell->subdomain_id(); const unsigned int dofs_per_cell = cell->get_fe().dofs_per_cell; local_dof_indices.resize (dofs_per_cell); cell->get_dof_indices (local_dof_indices); @@ -4289,7 +4541,7 @@ DoFTools::get_subdomain_association (const DH &dof_handler, Assert (std::find (subdomain_association.begin(), subdomain_association.end(), - numbers::invalid_unsigned_int) + types::invalid_subdomain_id) == subdomain_association.end(), ExcInternalError()); } @@ -4299,9 +4551,9 @@ DoFTools::get_subdomain_association (const DH &dof_handler, template unsigned int DoFTools::count_dofs_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain) + const types::subdomain_id_t subdomain) { - std::vector subdomain_association (dof_handler.n_dofs()); + std::vector subdomain_association (dof_handler.n_dofs()); get_subdomain_association (dof_handler, subdomain_association); return std::count (subdomain_association.begin(), @@ -4314,31 +4566,43 @@ DoFTools::count_dofs_with_subdomain_association (const DH &dof_handler template IndexSet DoFTools::dof_indices_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain) + const types::subdomain_id_t subdomain) { - std::vector subdomain_association (dof_handler.n_dofs()); - get_subdomain_association (dof_handler, subdomain_association); + // if the DoFHandler is distributed, the + // only thing we can usefully ask is for + // its locally owned subdomain + Assert ((dynamic_cast*> + (&dof_handler.get_tria()) == 0) + || + (subdomain == + dynamic_cast*> + (&dof_handler.get_tria())->locally_owned_subdomain()), + ExcMessage ("For parallel::distributed::Triangulation objects and " + "associated DoF handler objects, asking for any subdomain other " + "than the locally owned one does not make sense.")); IndexSet index_set (dof_handler.n_dofs()); - unsigned int index = 0; - while (index < dof_handler.n_dofs()) - { - // find first_of - while ((index < dof_handler.n_dofs()) && - (subdomain_association[index] != subdomain)) - ++index; - const unsigned int begin = index; - - // find first_not_of - while ((index < dof_handler.n_dofs()) && - (subdomain_association[index] == subdomain)) - ++index; - const unsigned int end = index; - - if ((begin < end) && (begin < dof_handler.n_dofs())) - index_set.add_range (begin, end); - } + std::vector local_dof_indices; + local_dof_indices.reserve (max_dofs_per_cell(dof_handler)); + + typename DH::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + for (; cell!=endc; ++cell) + if ((cell->is_artificial() == false) + && + (cell->subdomain_id() == subdomain)) + { + const unsigned int dofs_per_cell = cell->get_fe().dofs_per_cell; + local_dof_indices.resize (dofs_per_cell); + cell->get_dof_indices (local_dof_indices); + index_set.add_indices (local_dof_indices.begin(), + local_dof_indices.end()); + } + index_set.compress (); return index_set; } @@ -4351,7 +4615,7 @@ DoFTools::dof_indices_with_subdomain_association (const DH &dof_handle template void DoFTools::count_dofs_with_subdomain_association (const DH &dof_handler, - const unsigned int subdomain, + const types::subdomain_id_t subdomain, std::vector &n_dofs_on_subdomain) { Assert (n_dofs_on_subdomain.size() == dof_handler.get_fe().n_components(), @@ -4481,7 +4745,7 @@ DoFTools::count_dofs_per_component ( // computations if (n_components == 1) { - dofs_per_component[0] = dof_handler.n_dofs(); + dofs_per_component[0] = dof_handler.n_locally_owned_dofs(); return; } @@ -4489,7 +4753,7 @@ DoFTools::count_dofs_per_component ( // otherwise determine the number // of dofs in each component // separately. do so in parallel - std::vector dofs_by_component (dof_handler.n_dofs()); + std::vector dofs_by_component (dof_handler.n_locally_owned_dofs()); internal::extract_dofs_by_component (dof_handler, std::vector(), false, dofs_by_component); @@ -4507,11 +4771,25 @@ DoFTools::count_dofs_per_component ( || (std::accumulate (dofs_per_component.begin(), dofs_per_component.end(), 0U) - == dof_handler.n_dofs()), + == dof_handler.n_locally_owned_dofs()), ExcInternalError()); + + // reduce information from all CPUs +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + if (const parallel::distributed::Triangulation * tria + = (dynamic_cast*> + (&dof_handler.get_tria()))) + { + std::vector local_dof_count = dofs_per_component; + + MPI_Allreduce ( &local_dof_count[0], &dofs_per_component[0], n_target_components, + MPI_INT, MPI_SUM, tria->get_communicator()); + } +#endif } + template void DoFTools:: @@ -4558,19 +4836,35 @@ count_dofs_per_block (const DoFHandler& dof_handler, } // otherwise determine the number // of dofs in each block - // separately. do so in parallel - std::vector dofs_by_block (dof_handler.n_dofs()); + // separately. + std::vector dofs_by_block (dof_handler.n_locally_owned_dofs()); internal::extract_dofs_by_component (dof_handler, std::vector(), true, dofs_by_block); // next count what we got - for (unsigned int block=0;block * tria + = (dynamic_cast*> + (&dof_handler.get_tria()))) + { + std::vector local_dof_count = dofs_per_block; + MPI_Allreduce ( &local_dof_count[0], &dofs_per_block[0], n_target_blocks, + MPI_INT, MPI_SUM, tria->get_communicator()); + } +#endif } + template void DoFTools:: @@ -5770,7 +6064,7 @@ DoFTools::make_zero_boundary_constraints (const DH &dof, ++face_no) { const FiniteElement &fe = cell->get_fe(); - + typename DH::face_iterator face = cell->face(face_no); if (face->boundary_indicator () == 0) // face is of the right component @@ -5799,7 +6093,7 @@ DoFTools::make_zero_boundary_constraints (const DH &dof, nonzero = true; break; } - + if (nonzero) zero_boundary_constraints.add_line (face_dofs[i]); } @@ -6080,15 +6374,34 @@ template void DoFTools::extract_subdomain_dofs > (const DoFHandler &dof_handler, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, std::vector &selected_dofs); template void DoFTools::extract_subdomain_dofs > (const hp::DoFHandler &dof_handler, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, std::vector &selected_dofs); +template +void +DoFTools::extract_locally_owned_dofs > +(const DoFHandler & dof_handler, + IndexSet & dof_set); + +template +void +DoFTools::extract_locally_active_dofs > +(const DoFHandler & dof_handler, + IndexSet & dof_set); + + +template +void +DoFTools::extract_locally_relevant_dofs > +(const DoFHandler & dof_handler, + IndexSet & dof_set); + template void DoFTools::extract_constant_modes > @@ -6112,63 +6425,63 @@ template void DoFTools::get_subdomain_association > (const DoFHandler &dof_handler, - std::vector &subdomain_association); + std::vector &subdomain_association); template void DoFTools::get_subdomain_association > (const hp::DoFHandler &dof_handler, - std::vector &subdomain_association); + std::vector &subdomain_association); template unsigned int DoFTools::count_dofs_with_subdomain_association > (const DoFHandler &, - const unsigned int); + const types::subdomain_id_t); template IndexSet DoFTools::dof_indices_with_subdomain_association > (const DoFHandler &, - const unsigned int); + const types::subdomain_id_t); template void DoFTools::count_dofs_with_subdomain_association > (const DoFHandler &, - const unsigned int, + const types::subdomain_id_t, std::vector &); template unsigned int DoFTools::count_dofs_with_subdomain_association > (const hp::DoFHandler &, - const unsigned int); + const types::subdomain_id_t); template IndexSet DoFTools::dof_indices_with_subdomain_association > (const hp::DoFHandler &, - const unsigned int); + const types::subdomain_id_t); template void DoFTools::count_dofs_with_subdomain_association > (const hp::DoFHandler &, - const unsigned int, + const types::subdomain_id_t, std::vector &); template unsigned int DoFTools::count_dofs_with_subdomain_association > (const MGDoFHandler &, - const unsigned int); + const types::subdomain_id_t); template IndexSet DoFTools::dof_indices_with_subdomain_association > (const MGDoFHandler &, - const unsigned int); + const types::subdomain_id_t); template void DoFTools::count_dofs_with_subdomain_association > (const MGDoFHandler &, - const unsigned int, + const types::subdomain_id_t, std::vector &); @@ -6196,6 +6509,7 @@ DoFTools::count_dofs_per_block ( const DoFHandler&, std::vector&, std::vector); + template void DoFTools::count_dofs_per_component ( diff --git a/deal.II/deal.II/source/dofs/number_cache.cc b/deal.II/deal.II/source/dofs/number_cache.cc new file mode 100644 index 0000000000..bff02e10d4 --- /dev/null +++ b/deal.II/deal.II/source/dofs/number_cache.cc @@ -0,0 +1,45 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + + +#include +#include + +DEAL_II_NAMESPACE_OPEN + +namespace internal +{ + namespace DoFHandler + { + NumberCache::NumberCache () + : + n_global_dofs (0), + n_locally_owned_dofs (0) + {} + + + + unsigned int + NumberCache::memory_consumption () const + { + return + MemoryConsumption::memory_consumption (n_global_dofs) + + MemoryConsumption::memory_consumption (n_locally_owned_dofs) + + MemoryConsumption::memory_consumption (locally_owned_dofs) + + MemoryConsumption::memory_consumption (n_locally_owned_dofs_per_processor) + + MemoryConsumption::memory_consumption (locally_owned_dofs_per_processor); + } + } +} + +DEAL_II_NAMESPACE_CLOSE diff --git a/deal.II/deal.II/source/fe/fe_tools.cc b/deal.II/deal.II/source/fe/fe_tools.cc index 25fbdcb63e..ec1298b4fc 100644 --- a/deal.II/deal.II/source/fe/fe_tools.cc +++ b/deal.II/deal.II/source/fe/fe_tools.cc @@ -11,8 +11,6 @@ // //--------------------------------------------------------------------------- -#include - #include #include #include @@ -47,6 +45,8 @@ #include +#include + DEAL_II_NAMESPACE_OPEN diff --git a/deal.II/deal.II/source/grid/grid_refinement.cc b/deal.II/deal.II/source/grid/grid_refinement.cc index c96a80f034..8e36fa9af3 100644 --- a/deal.II/deal.II/source/grid/grid_refinement.cc +++ b/deal.II/deal.II/source/grid/grid_refinement.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors +// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -11,9 +11,14 @@ // //--------------------------------------------------------------------------- +#include #include +#include +#include #include +#include #include +#include #include #include @@ -29,73 +34,121 @@ DEAL_II_NAMESPACE_OPEN -namespace +namespace { - template - inline - number - max_element (const Vector &criteria) + namespace internal { - return *std::max_element(criteria.begin(), criteria.end()); - } + template + inline + number + max_element (const Vector &criteria) + { + return *std::max_element(criteria.begin(), criteria.end()); + } - - template - inline - number - min_element (const Vector &criteria) - { - return *std::min_element(criteria.begin(), criteria.end()); + + template + inline + number + min_element (const Vector &criteria) + { + return *std::min_element(criteria.begin(), criteria.end()); + } + + +#ifdef DEAL_II_USE_PETSC + PetscScalar + max_element (const PETScWrappers::Vector &criteria) + { + // this is horribly slow (since we have + // to get the array of values from PETSc + // in every iteration), but works + PetscScalar m = 0; + for (unsigned int i=0; i + typename constraint_and_return_value::value, + typename Vector::value_type>::type + min_element (const Vector &criteria) { - // this is horribly slow (since we have - // to get the array of values from PETSc - // in every iteration), but works - PetscScalar m = 0; - for (unsigned int i=0; i + typename constraint_and_return_value::value, + typename Vector::value_type>::type + max_element (const Vector &criteria) { - // this is horribly slow (since we have - // to get the array of values from PETSc - // in every iteration), but works - PetscScalar m = criteria(0); - for (unsigned int i=1; i + typename constraint_and_return_value::value, + typename Vector::value_type>::type + min_element (const Vector &criteria) { - TrilinosScalar m = 0; - criteria.trilinos_vector().MaxValue(&m); - return m; + typename Vector::value_type t = internal::min_element(criteria.block(0)); + for (unsigned int b=1; b + typename constraint_and_return_value::value, + typename Vector::value_type>::type + max_element (const Vector &criteria) { - TrilinosScalar m = 0; - criteria.trilinos_vector().MinValue(&m); - return m; + typename Vector::value_type t = internal::max_element(criteria.block(0)); + for (unsigned int b=1; b0)); - + if (i &tria, Assert (criteria.size() == tria.n_active_cells(), ExcDimensionMismatch(criteria.size(), tria.n_active_cells())); Assert (criteria.is_non_negative (), ExcNegativeCriteria()); - + // when all indicators are zero we // do not need to refine but only // to coarsen if (criteria.all_zero()) return; - + typename Triangulation::active_cell_iterator cell = tria.begin_active(); const unsigned int n_cells = criteria.size(); //TODO: This is undocumented, looks fishy and seems unnecessary - + double new_threshold=threshold; // when threshold==0 find the // smallest value in criteria @@ -185,7 +238,7 @@ void GridRefinement::refine (Triangulation &tria, && (criteria(index)= new_threshold) cell->set_refine_flag(); @@ -204,7 +257,7 @@ void GridRefinement::coarsen (Triangulation &tria, typename Triangulation::active_cell_iterator cell = tria.begin_active(); const unsigned int n_cells = criteria.size(); - + for (unsigned int index=0; indexrefine_flag_set()) @@ -240,7 +293,7 @@ GridRefinement::refine_and_coarsen_fixed_number (Triangulation &tr // refining cells and instead try to // only coarsen as many as it would // take to get to the target - + // as we have no information on cells // being refined isotropically or // anisotropically, assume isotropic @@ -293,10 +346,10 @@ GridRefinement::refine_and_coarsen_fixed_number (Triangulation &tr refine_cells = static_cast (refine_cells * alpha); coarsen_cells = static_cast (coarsen_cells * alpha); } - + if (refine_cells || coarsen_cells) { - dealii::Vector tmp(criteria); + dealii::Vector tmp (criteria); if (refine_cells) { std::nth_element (tmp.begin(), tmp.begin()+refine_cells, @@ -332,12 +385,13 @@ GridRefinement::refine_and_coarsen_fixed_fraction (Triangulation & Assert ((bottom_fraction>=0) && (bottom_fraction<=1), ExcInvalidParameterValue()); Assert (top_fraction+bottom_fraction <= 1, ExcInvalidParameterValue()); Assert (criteria.is_non_negative (), ExcNegativeCriteria()); - + // let tmp be the cellwise square of the // error, which is what we have to sum // up and compare with // @p{fraction_of_error*total_error}. - dealii::Vector tmp(criteria); + dealii::Vector tmp; + tmp = criteria; const double total_error = tmp.l1_norm(); // sort the largest criteria to the @@ -397,7 +451,7 @@ GridRefinement::refine_and_coarsen_fixed_fraction (Triangulation & return; } } - + // in some rare cases it may happen that // both thresholds are the same (e.g. if @@ -433,14 +487,14 @@ GridRefinement::refine_and_coarsen_fixed_fraction (Triangulation & if ((top_threshold == max_element(criteria)) && (top_fraction != 1)) top_threshold *= 0.999; - + if (bottom_threshold>=top_threshold) bottom_threshold = 0.999*top_threshold; - + // actually flag cells if (top_threshold < max_element(criteria)) refine (tria, criteria, top_threshold); - + if (bottom_threshold > min_element(criteria)) coarsen (tria, criteria, bottom_threshold); } @@ -455,21 +509,21 @@ GridRefinement::refine_and_coarsen_optimize (Triangulation &tria, Assert (criteria.size() == tria.n_active_cells(), ExcDimensionMismatch(criteria.size(), tria.n_active_cells())); Assert (criteria.is_non_negative (), ExcNegativeCriteria()); - + // get an increasing order on // the error indicator std::vector tmp(criteria.size()); for (unsigned int i=0;i &tria, // N+3*M (N is the current number // of cells) double min = (3.*(1.+M)+N) * (E-s0); - + unsigned int minArg = N-1; - + for (M=1;M &tria, // explicit instantiations -template -void -GridRefinement:: -refine,deal_II_dimension> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -refine,deal_II_dimension> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -coarsen,deal_II_dimension> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -coarsen,deal_II_dimension> - (Triangulation &, - const Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number,deal_II_dimension> -(Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number,deal_II_dimension> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction,deal_II_dimension> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction,deal_II_dimension> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize,deal_II_dimension> -(Triangulation &, - const Vector &); - -template -void -GridRefinement:: -refine_and_coarsen_optimize,deal_II_dimension> -(Triangulation &, - const Vector &); - -#ifdef DEAL_II_USE_PETSC -template -void -GridRefinement:: -refine - (Triangulation &, - const PETScWrappers::Vector &, - const double); - -template -void -GridRefinement:: -coarsen -(Triangulation &, - const PETScWrappers::Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number -(Triangulation &, - const PETScWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction -(Triangulation &, - const PETScWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize -(Triangulation &, - const PETScWrappers::Vector &); -#endif - -#ifdef DEAL_II_USE_TRILINOS -template -void -GridRefinement:: -refine - (Triangulation &, - const TrilinosWrappers::Vector &, - const double); - -template -void -GridRefinement:: -coarsen -(Triangulation &, - const TrilinosWrappers::Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number -(Triangulation &, - const TrilinosWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction -(Triangulation &, - const TrilinosWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize -(Triangulation &, - const TrilinosWrappers::Vector &); -#endif - - - -#if deal_II_dimension != 3 -// explicit instantiations -template -void -GridRefinement:: -refine,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -refine,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -coarsen,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double); - -template -void -GridRefinement:: -coarsen,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number,deal_II_dimension+1> -(Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction,deal_II_dimension+1> - (Triangulation &, - const Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize,deal_II_dimension+1> -(Triangulation &, - const Vector &); - -template -void -GridRefinement:: -refine_and_coarsen_optimize,deal_II_dimension+1> -(Triangulation &, - const Vector &); - -#ifdef DEAL_II_USE_PETSC -template -void -GridRefinement:: -refine - (Triangulation &, - const PETScWrappers::Vector &, - const double); - -template -void -GridRefinement:: -coarsen -(Triangulation &, - const PETScWrappers::Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number -(Triangulation &, - const PETScWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction -(Triangulation &, - const PETScWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize -(Triangulation &, - const PETScWrappers::Vector &); -#endif - -#ifdef DEAL_II_USE_TRILINOS -template -void -GridRefinement:: -refine - (Triangulation &, - const TrilinosWrappers::Vector &, - const double); - -template -void -GridRefinement:: -coarsen -(Triangulation &, - const TrilinosWrappers::Vector &, - const double); - - -template -void -GridRefinement:: -refine_and_coarsen_fixed_number -(Triangulation &, - const TrilinosWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_fixed_fraction -(Triangulation &, - const TrilinosWrappers::Vector &, - const double, - const double, - const unsigned int); - -template -void -GridRefinement:: -refine_and_coarsen_optimize -(Triangulation &, - const TrilinosWrappers::Vector &); -#endif - - -#endif +#include "grid_refinement.inst" DEAL_II_NAMESPACE_CLOSE diff --git a/deal.II/deal.II/source/grid/grid_refinement.inst.in b/deal.II/deal.II/source/grid/grid_refinement.inst.in new file mode 100644 index 0000000000..d4619e799d --- /dev/null +++ b/deal.II/deal.II/source/grid/grid_refinement.inst.in @@ -0,0 +1,105 @@ +//--------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//--------------------------------------------------------------------------- + + + +for (S : REAL_SCALARS) +{ + template + void + GridRefinement:: + refine,deal_II_dimension> + (Triangulation &, + const dealii::Vector &, + const double); + + template + void + GridRefinement:: + coarsen,deal_II_dimension> + (Triangulation &, + const dealii::Vector &, + const double); + + template + void + GridRefinement:: + refine_and_coarsen_fixed_number,deal_II_dimension> + (Triangulation &, + const dealii::Vector &, + const double, + const double, + const unsigned int); + + template + void + GridRefinement:: + refine_and_coarsen_fixed_fraction,deal_II_dimension> + (Triangulation &, + const dealii::Vector &, + const double, + const double, + const unsigned int); + + template + void + GridRefinement:: + refine_and_coarsen_optimize,deal_II_dimension> + (Triangulation &, + const dealii::Vector &); + +#if deal_II_dimension < 3 + template + void + GridRefinement:: + refine,deal_II_dimension+1> + (Triangulation &, + const dealii::Vector &, + const double); + + template + void + GridRefinement:: + coarsen,deal_II_dimension+1> + (Triangulation &, + const dealii::Vector &, + const double); + + template + void + GridRefinement:: + refine_and_coarsen_fixed_number,deal_II_dimension+1> + (Triangulation &, + const dealii::Vector &, + const double, + const double, + const unsigned int); + + template + void + GridRefinement:: + refine_and_coarsen_fixed_fraction,deal_II_dimension+1> + (Triangulation &, + const dealii::Vector &, + const double, + const double, + const unsigned int); + + template + void + GridRefinement:: + refine_and_coarsen_optimize,deal_II_dimension+1> + (Triangulation &, + const dealii::Vector &); +#endif +} diff --git a/deal.II/deal.II/source/grid/grid_tools.cc b/deal.II/deal.II/source/grid/grid_tools.cc index d845d538cd..77b81cc56e 100644 --- a/deal.II/deal.II/source/grid/grid_tools.cc +++ b/deal.II/deal.II/source/grid/grid_tools.cc @@ -989,7 +989,7 @@ template void GridTools:: get_subdomain_association (const Triangulation &triangulation, - std::vector &subdomain) + std::vector &subdomain) { Assert (subdomain.size() == triangulation.n_active_cells(), ExcDimensionMismatch (subdomain.size(), @@ -1009,7 +1009,7 @@ template unsigned int GridTools:: count_cells_with_subdomain_association (const Triangulation &triangulation, - const unsigned int subdomain) + const types::subdomain_id_t subdomain) { unsigned int count = 0; for (typename Triangulation::active_cell_iterator @@ -1018,8 +1018,6 @@ count_cells_with_subdomain_association (const Triangulation &tria if (cell->subdomain_id() == subdomain) ++count; - Assert (count != 0, ExcNonExistentSubdomain(subdomain)); - return count; } @@ -1863,6 +1861,14 @@ GridTools:: get_face_connectivity_of_cells (const Triangulation &triangulation, SparsityPattern &cell_connectivity); +#if deal_II_dimension < 3 +template +void +GridTools:: +get_face_connectivity_of_cells (const Triangulation &triangulation, + SparsityPattern &cell_connectivity); +#endif + template void GridTools::partition_triangulation (const unsigned int, @@ -1878,13 +1884,13 @@ template void GridTools:: get_subdomain_association (const Triangulation &, - std::vector &); + std::vector &); template unsigned int GridTools:: count_cells_with_subdomain_association (const Triangulation &, - const unsigned int ); + const types::subdomain_id_t); template diff --git a/deal.II/deal.II/source/grid/tria.cc b/deal.II/deal.II/source/grid/tria.cc index 7e13a79f6b..6e390afd5f 100644 --- a/deal.II/deal.II/source/grid/tria.cc +++ b/deal.II/deal.II/source/grid/tria.cc @@ -4042,6 +4042,8 @@ namespace internal subcells[i]->set_parent (cell->index ()); } + + // set child index for // even children children // i=0,2 (0) @@ -9152,9 +9154,9 @@ Triangulation:: Triangulation (const MeshSmoothing smooth_grid, const bool check_for_distorted_cells) : + smooth_grid(smooth_grid), faces(NULL), anisotropic_refinement(false), - smooth_grid(smooth_grid), check_for_distorted_cells(check_for_distorted_cells) { // set default boundary for all @@ -9178,6 +9180,7 @@ Triangulation (const Triangulation &) } + template Triangulation::~Triangulation () { @@ -9188,6 +9191,7 @@ Triangulation::~Triangulation () } + template void Triangulation::clear () { @@ -11648,6 +11652,8 @@ unsigned int Triangulation::n_lines () const #if deal_II_dimension == 1 + + template <> unsigned int Triangulation<1,1>::n_raw_lines (const unsigned int level) const { @@ -12013,6 +12019,7 @@ Triangulation::get_used_vertices () const #if deal_II_dimension == 1 + template <> unsigned int Triangulation<1,1>::max_adjacent_cells () const { @@ -12060,6 +12067,13 @@ Triangulation::execute_coarsening_and_refinement () { prepare_coarsening_and_refinement (); + // verify a case with which we have had + // some difficulty in the past (see the + // deal.II/coarsening_* tests) + if (smooth_grid & limit_level_difference_at_vertices) + Assert (satisfies_level1_at_vertex_rule (*this) == true, + ExcInternalError()); + // Inform RefinementListeners // about beginning of refinement. typename std::list::iterator ref_listener = @@ -12071,8 +12085,7 @@ Triangulation::execute_coarsening_and_refinement () execute_coarsening(); const DistortedCellList - cells_with_distorted_children - = execute_refinement(); + cells_with_distorted_children = execute_refinement(); // verify a case with which we have had // some difficulty in the past (see the @@ -12274,6 +12287,8 @@ void Triangulation::fix_coarsen_flags () std::vector previous_coarsen_flags (n_active_cells()); save_coarsen_flags (previous_coarsen_flags); + std::vector vertex_level (vertices.size(), 0); + bool continue_iterating = true; do @@ -12288,7 +12303,7 @@ void Triangulation::fix_coarsen_flags () // store highest level one // of the cells adjacent to // a vertex belongs to - std::vector vertex_level (vertices.size(), 0); + std::fill (vertex_level.begin(), vertex_level.end(), 0); active_cell_iterator cell = begin_active(), endc = end(); for (; cell!=endc; ++cell) @@ -12553,6 +12568,62 @@ bool Triangulation<1,2>::prepare_coarsening_and_refinement () namespace { + + // check if the given @param cell marked + // for coarsening would produce an + // unrefined island. To break up long + // chains of these cells we recursively + // check our neighbors in case we change + // this cell. This reduces the number of + // outer iterations dramatically. + template + void + possibly_do_not_produce_unrefined_islands( + const typename Triangulation::cell_iterator &cell) + { + Assert (cell->has_children(), ExcInternalError()); + + unsigned int n_neighbors=0; + // count all neighbors + // that will be refined + // along the face of our + // cell after the next + // step + unsigned int count=0; + for (unsigned int n=0; n::faces_per_cell; ++n) + { + const typename Triangulation::cell_iterator neighbor = cell->neighbor(n); + if (neighbor.state() == IteratorState::valid) + { + ++n_neighbors; + if (face_will_be_refined_by_neighbor(cell,n)) + ++count; + } + } + // clear coarsen flags if + // either all existing + // neighbors will be + // refined or all but one + // will be and the cell + // is in the interior of + // the domain + if (count==n_neighbors || + (count>=n_neighbors-1 && + n_neighbors == GeometryInfo::faces_per_cell) ) + { + for (unsigned int c=0; cn_children(); ++c) + cell->child(c)->clear_coarsen_flag(); + + for (unsigned int face=0; face::faces_per_cell; ++face) + if (!cell->at_boundary(face) + && + ( !cell->neighbor(face)->active() ) + && (cell_will_be_coarsened(cell->neighbor(face))) ) + possibly_do_not_produce_unrefined_islands( cell->neighbor(face) ); + } + } + + // see if the current cell needs to // be refined to avoid unrefined // islands. @@ -12926,11 +12997,9 @@ bool Triangulation::prepare_coarsening_and_refinement () cell->clear_coarsen_flag(); } - bool mesh_changed_in_this_loop = false; do { - ////////////////////////////////////// // STEP 1: // do not coarsen a cell if 'most of @@ -12947,47 +13016,12 @@ bool Triangulation::prepare_coarsening_and_refinement () for (cell=begin(); cell!=endc; ++cell) { - if (!cell->active()) - { // only do something if this // cell will be coarsened - if (cell_will_be_coarsened(cell)) - { - unsigned int n_neighbors=0; - // count all neighbors - // that will be refined - // along the face of our - // cell after the next - // step - unsigned int count=0; - for (unsigned int n=0; - n::faces_per_cell; ++n) - { - const cell_iterator neighbor = cell->neighbor(n); - if (neighbor.state() == IteratorState::valid) - { - ++n_neighbors; - if (face_will_be_refined_by_neighbor(cell,n)) - ++count; - } - } - // clear coarsen flags if - // either all existing - // neighbors will be - // refined or all but one - // will be and the cell - // is in the interior of - // the domain - if (count==n_neighbors || - (count>=n_neighbors-1 && - n_neighbors== - GeometryInfo::faces_per_cell)) - for (unsigned int c=0; cn_children(); ++c) - cell->child(c)->clear_coarsen_flag(); - } - } // if (!cell->active()) - } // for (all cells) - } // if (smooth_grid & ...) + if (!cell->active() && cell_will_be_coarsened(cell)) + possibly_do_not_produce_unrefined_islands(cell); + } + } ////////////////////////////////////// @@ -13000,6 +13034,31 @@ bool Triangulation::prepare_coarsening_and_refinement () // priority. // If patch_level_1 is set, this will // be automatically fulfilled. + // + // there is one corner case + // to consider: if this is a + // distributed + // triangulation, there may + // be refined islands on the + // boundary of which we own + // only part (e.g. a single + // cell in the corner of a + // domain). the rest of the + // island is ghost cells and + // it *looks* like the area + // around it (artificial + // cells) are coarser but + // this is only because they + // may actually be equally + // fine on other + // processors. it's hard to + // detect this case but we + // can do the following: + // only set coarsen flags to + // remove this refined + // island if all cells we + // want to set flags on are + // locally owned if (smooth_grid & (eliminate_refined_inner_islands | eliminate_refined_boundary_islands) && !(smooth_grid & patch_level_1)) @@ -13008,7 +13067,11 @@ bool Triangulation::prepare_coarsening_and_refinement () const cell_iterator endc = end(); for (cell=begin(); cell!=endc; ++cell) - if (!cell->active() || (cell->active() && cell->refine_flag_set())) + if (!cell->active() || + (cell->active() && + cell->refine_flag_set() && + !cell->is_ghost() && + !cell->is_artificial())) { // check whether all // children are @@ -13027,7 +13090,9 @@ bool Triangulation::prepare_coarsening_and_refinement () bool all_children_active = true; if (!cell->active()) for (unsigned int c=0; cn_children(); ++c) - if (!cell->child(c)->active()) + if (!cell->child(c)->active() || + cell->child(c)->is_ghost() || + cell->child(c)->is_artificial()) { all_children_active = false; break; diff --git a/deal.II/deal.II/source/grid/tria_accessor.cc b/deal.II/deal.II/source/grid/tria_accessor.cc index b8c5d20815..4d9214845e 100644 --- a/deal.II/deal.II/source/grid/tria_accessor.cc +++ b/deal.II/deal.II/source/grid/tria_accessor.cc @@ -1264,7 +1264,7 @@ void CellAccessor::recursively_set_material_id (const unsigned ch template -unsigned int CellAccessor::subdomain_id () const +types::subdomain_id_t CellAccessor::subdomain_id () const { Assert (this->used(), TriaAccessorExceptions::ExcCellNotUsed()); return this->tria->levels[this->present_level]->subdomain_ids[this->present_index]; @@ -1274,7 +1274,7 @@ unsigned int CellAccessor::subdomain_id () const template void -CellAccessor::set_subdomain_id (const unsigned int new_subdomain_id) const +CellAccessor::set_subdomain_id (const types::subdomain_id_t new_subdomain_id) const { Assert (this->used(), TriaAccessorExceptions::ExcCellNotUsed()); this->tria->levels[this->present_level]->subdomain_ids[this->present_index] @@ -1295,6 +1295,20 @@ CellAccessor::parent () const } +template +void +CellAccessor:: +recursively_set_subdomain_id (const types::subdomain_id_t new_subdomain_id) const +{ + set_subdomain_id (new_subdomain_id); + + if (this->has_children()) + for (unsigned int c=0; cn_children(); ++c) + this->child(c)->recursively_set_subdomain_id (new_subdomain_id); +} + + + template void CellAccessor::set_neighbor (const unsigned int i, const TriaIterator > &pointer) const diff --git a/deal.II/deal.II/source/hp/dof_handler.cc b/deal.II/deal.II/source/hp/dof_handler.cc index 7513dae2be..2995838cdf 100644 --- a/deal.II/deal.II/source/hp/dof_handler.cc +++ b/deal.II/deal.II/source/hp/dof_handler.cc @@ -31,6 +31,15 @@ DEAL_II_NAMESPACE_OPEN +namespace parallel +{ + namespace distributed + { + template class Triangulation; + } +} + + namespace internal { namespace hp @@ -1587,9 +1596,14 @@ namespace hp DoFHandler::DoFHandler (const Triangulation &tria) : tria(&tria, typeid(*this).name()), - faces (NULL), - used_dofs (0) - { + faces (NULL) + { + Assert ((dynamic_cast*> + (&tria) + == 0), + ExcMessage ("The given triangulation is parallel distributed but " + "this class does not currently support this.")); + create_active_fe_table (); tria.add_refinement_listener (*this); } @@ -2924,7 +2938,7 @@ namespace hp MemoryConsumption::memory_consumption (tria) + MemoryConsumption::memory_consumption (levels) + MemoryConsumption::memory_consumption (*faces) + - MemoryConsumption::memory_consumption (used_dofs) + + MemoryConsumption::memory_consumption (number_cache) + MemoryConsumption::memory_consumption (vertex_dofs) + MemoryConsumption::memory_consumption (vertex_dofs_offsets) + MemoryConsumption::memory_consumption (has_children)); @@ -3491,7 +3505,7 @@ namespace hp = internal::hp::DoFHandler::Implementation::template distribute_dofs_on_cell (cell, next_free_dof); - used_dofs = next_free_dof; + number_cache.n_global_dofs = next_free_dof; } @@ -3505,7 +3519,7 @@ namespace hp // lower-dimensional objects // where elements come together std::vector - constrained_indices (used_dofs, numbers::invalid_unsigned_int); + constrained_indices (number_cache.n_global_dofs, numbers::invalid_unsigned_int); compute_vertex_dof_identities (constrained_indices); compute_line_dof_identities (constrained_indices); compute_quad_dof_identities (constrained_indices); @@ -3514,9 +3528,9 @@ namespace hp // new numbers to those which are // not constrained std::vector - new_dof_indices (used_dofs, numbers::invalid_unsigned_int); + new_dof_indices (number_cache.n_global_dofs, numbers::invalid_unsigned_int); unsigned int next_free_dof = 0; - for (unsigned int i=0; i()); - used_dofs = next_free_dof; + // now set the elements of the + // number cache appropriately + number_cache.n_global_dofs = next_free_dof; + number_cache.n_locally_owned_dofs = number_cache.n_global_dofs; + + number_cache.locally_owned_dofs + = IndexSet (number_cache.n_global_dofs); + number_cache.locally_owned_dofs.add_range (0, + number_cache.n_global_dofs); + + number_cache.n_locally_owned_dofs_per_processor + = std::vector (1, + number_cache.n_global_dofs); + number_cache.locally_owned_dofs_per_processor + = std::vector (1, + number_cache.locally_owned_dofs); // finally restore the user flags const_cast &>(*tria).load_user_flags(user_flags); diff --git a/deal.II/deal.II/source/multigrid/mg_dof_handler.cc b/deal.II/deal.II/source/multigrid/mg_dof_handler.cc index 7ec46d3036..cc03d6bdf3 100644 --- a/deal.II/deal.II/source/multigrid/mg_dof_handler.cc +++ b/deal.II/deal.II/source/multigrid/mg_dof_handler.cc @@ -30,6 +30,14 @@ DEAL_II_NAMESPACE_OPEN +namespace parallel +{ + namespace distributed + { + template class Triangulation; + } +} + //TODO[WB]: this class is currently only implemented for dim==spacedim. to //make the general case happen it should undergo a similar transformation as @@ -489,7 +497,8 @@ namespace internal { const unsigned int dim = 1; - Assert (mg_dof_handler.get_tria().n_levels() > 0, DoFHandler<1>::ExcInvalidTriangulation()); + Assert (mg_dof_handler.get_tria().n_levels() > 0, + ExcMessage("Invalid triangulation")); ////////////////////////// // DESTRUCTION @@ -567,7 +576,8 @@ namespace internal void reserve_space (MGDoFHandler<2,spacedim> &mg_dof_handler) { const unsigned int dim = 2; - Assert (mg_dof_handler.get_tria().n_levels() > 0, DoFHandler<2>::ExcInvalidTriangulation()); + Assert (mg_dof_handler.get_tria().n_levels() > 0, + ExcMessage("Invalid triangulation")); //////////////////////////// // DESTRUCTION @@ -656,7 +666,8 @@ namespace internal { const unsigned int dim = 3; - Assert (mg_dof_handler.get_tria().n_levels() > 0, DoFHandler<3>::ExcInvalidTriangulation()); + Assert (mg_dof_handler.get_tria().n_levels() > 0, + ExcMessage("Invalid triangulation")); //////////////////////////// // DESTRUCTION @@ -831,7 +842,13 @@ MGDoFHandler::MGDoFHandler (const Triangulation &tri : DoFHandler (tria), mg_faces (NULL) -{} +{ + Assert ((dynamic_cast*> + (&tria) + == 0), + ExcMessage ("The given triangulation is parallel distributed but " + "this class does not currently support this.")); +} diff --git a/deal.II/deal.II/source/numerics/data_out.cc b/deal.II/deal.II/source/numerics/data_out.cc index 0926a72d99..9a346cbf82 100644 --- a/deal.II/deal.II/source/numerics/data_out.cc +++ b/deal.II/deal.II/source/numerics/data_out.cc @@ -970,8 +970,8 @@ void DataOut::build_patches (const Mappingdofs->get_tria().n_levels(); ++l) { unsigned int max_index = 0; - for (cell_iterator cell=first_cell(); cell != this->dofs->end(); - cell = next_cell(cell)) + for (cell_iterator cell=first_locally_owned_cell(); cell != this->dofs->end(); + cell = next_locally_owned_cell(cell)) if (static_cast(cell->level()) == l) max_index = std::max (max_index, static_cast(cell->index())); @@ -983,19 +983,19 @@ void DataOut::build_patches (const Mapping > all_cells; { // set the index of the first - // cell. if first_cell/next_cell + // cell. if first_locally_owned_cell/next_locally_owned_cell // returns non-active cells, then // the index is not usable // anyway, but otherwise we // should keep track where we are unsigned int index; - if (first_cell()->has_children()) + if (first_locally_owned_cell()->has_children()) index = 0; else index = std::distance (this->dofs->begin_active(), - active_cell_iterator(first_cell())); - for (cell_iterator cell=first_cell(); cell != this->dofs->end(); - cell = next_cell(cell)) + active_cell_iterator(first_locally_owned_cell())); + for (cell_iterator cell=first_locally_owned_cell(); cell != this->dofs->end(); + cell = next_locally_owned_cell(cell)) { Assert (static_cast(cell->level()) < cell_to_patch_index_map.size(), @@ -1020,10 +1020,10 @@ void DataOut::build_patches (const Mappinghas_children() && - next_cell(cell) != this->dofs->end() && - !next_cell(cell)->has_children()) + next_locally_owned_cell(cell) != this->dofs->end() && + !next_locally_owned_cell(cell)->has_children()) index += std::distance (active_cell_iterator(cell), - active_cell_iterator(next_cell(cell))); + active_cell_iterator(next_locally_owned_cell(cell))); } } @@ -1117,6 +1117,41 @@ DataOut::next_cell (const typename DataOut::cell_iterator &cell) } + +template +typename DataOut::cell_iterator +DataOut::first_locally_owned_cell () +{ + typename DataOut::cell_iterator + cell = this->dofs->begin_active (); + + // skip cells if the current one + // has no children (is active) and + // is a ghost or artificial cell + while ((cell != this->dofs->end()) && + (cell->has_children() == false) && + (cell->is_ghost() || cell->is_artificial())) + cell = next_cell(cell); + + return cell; +} + + + +template +typename DataOut::cell_iterator +DataOut::next_locally_owned_cell (const typename DataOut::cell_iterator &old_cell) +{ + typename DataOut::cell_iterator + cell = next_cell(old_cell); + while ((cell != this->dofs->end()) && + (cell->has_children() == false) && + (cell->is_ghost() || cell->is_artificial())) + cell = next_cell(cell); + return cell; +} + + // explicit instantiations #include "data_out.inst" diff --git a/deal.II/deal.II/source/numerics/data_out_faces.cc b/deal.II/deal.II/source/numerics/data_out_faces.cc index 3b4b62e340..8760e59d56 100644 --- a/deal.II/deal.II/source/numerics/data_out_faces.cc +++ b/deal.II/deal.II/source/numerics/data_out_faces.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors +// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -87,6 +87,10 @@ build_one_patch (const FaceDescriptor *cell_and_face, internal::DataOutFaces::ParallelData &data, DataOutBase::Patch &patch) { + Assert (!cell_and_face->first->is_ghost() && + !cell_and_face->first->is_artificial(), + ExcNotImplemented()); + // we use the mapping to transform the // vertices. However, the mapping works on // cells, not faces, so transform the face diff --git a/deal.II/deal.II/source/numerics/data_out_rotation.cc b/deal.II/deal.II/source/numerics/data_out_rotation.cc index 148c5b8627..94a0e8f587 100644 --- a/deal.II/deal.II/source/numerics/data_out_rotation.cc +++ b/deal.II/deal.II/source/numerics/data_out_rotation.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors +// Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -103,6 +103,10 @@ build_one_patch (const cell_iterator *cell, internal::DataOutRotation::ParallelData &data, std::vector > &patches) { + Assert (!(*cell)->is_ghost() && + !(*cell)->is_artificial(), + ExcNotImplemented()); + const unsigned int n_patches_per_circle = data.n_patches_per_circle; // another abbreviation denoting diff --git a/deal.II/deal.II/source/numerics/data_out_stack.cc b/deal.II/deal.II/source/numerics/data_out_stack.cc index 048e6ebda8..8c24e7c3a9 100644 --- a/deal.II/deal.II/source/numerics/data_out_stack.cc +++ b/deal.II/deal.II/source/numerics/data_out_stack.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006 by the deal.II authors +// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -302,6 +302,10 @@ void DataOutStack::build_patches (const unsigned int nnnn_subdi for (typename DH::active_cell_iterator cell=dof_handler->begin_active(); cell != dof_handler->end(); ++cell, ++patch, ++cell_number) { + Assert (!cell->is_ghost() && + !cell->is_artificial(), + ExcNotImplemented()); + Assert (patch != patches.end(), ExcInternalError()); // first fill in the vertices of the patch diff --git a/deal.II/deal.II/source/numerics/error_estimator.cc b/deal.II/deal.II/source/numerics/error_estimator.cc index cfa015a050..fac5c18221 100644 --- a/deal.II/deal.II/source/numerics/error_estimator.cc +++ b/deal.II/deal.II/source/numerics/error_estimator.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors +// Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -226,7 +226,7 @@ namespace internal * The subdomain id we are to care * for. */ - const unsigned int subdomain_id; + const types::subdomain_id_t subdomain_id; /** * The material id we are to care * for. @@ -251,7 +251,7 @@ namespace internal const dealii::hp::MappingCollection &mapping, const bool need_quadrature_points, const unsigned int n_solution_vectors, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id, const typename FunctionMap::type *neumann_bc, const std::vector component_mask, @@ -275,7 +275,7 @@ namespace internal const dealii::hp::MappingCollection &mapping, const bool need_quadrature_points, const unsigned int n_solution_vectors, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id, const typename FunctionMap::type *neumann_bc, const std::vector component_mask, @@ -383,6 +383,7 @@ namespace internal // exists in the global map Assert (face_integrals.find (p->first) == face_integrals.end(), ExcInternalError()); + for (unsigned int i=0; isecond.size(); ++i) Assert (p->second[i] >= 0, ExcInternalError()); @@ -828,7 +829,7 @@ namespace internal { const unsigned int n_solution_vectors = solutions.size(); - const unsigned int subdomain_id = parallel_data.subdomain_id; + const types::subdomain_id_t subdomain_id = parallel_data.subdomain_id; const unsigned int material_id = parallel_data.material_id; // empty our own copy of the local face @@ -901,7 +902,7 @@ namespace internal // material_id), or if one of the // neighbors behind the face is on // the subdomain we care for - if ( ! ( ((subdomain_id == numbers::invalid_unsigned_int) + if ( ! ( ((subdomain_id == types::invalid_subdomain_id) || (cell->subdomain_id() == subdomain_id)) && @@ -921,7 +922,7 @@ namespace internal if (face->has_children() == false) care_for_cell |= ((cell->neighbor(face_no)->subdomain_id() == subdomain_id) || - (subdomain_id == numbers::invalid_unsigned_int)) + (subdomain_id == types::invalid_subdomain_id)) && ((cell->neighbor(face_no)->material_id() == material_id) || @@ -939,7 +940,7 @@ namespace internal ->material_id() == material_id) && (subdomain_id == - numbers::invalid_unsigned_int))) + types::invalid_subdomain_id))) { care_for_cell = true; break; @@ -1015,7 +1016,7 @@ estimate (const Mapping<1,spacedim> &mapping, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { // just pass on to the other function @@ -1039,7 +1040,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1061,7 +1062,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1084,7 +1085,7 @@ estimate (const Mapping<1,spacedim> &mapping, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { // just pass on to the other function @@ -1107,7 +1108,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1129,7 +1130,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function<1> *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1152,7 +1153,7 @@ estimate (const Mapping<1,spacedim> &/*mapping*/, const std::vector &/*component_mask_*/, const Function<1> */*coefficient*/, const unsigned int, - const unsigned int /*subdomain_id*/, + const types::subdomain_id_t /*subdomain_id*/, const unsigned int /*material_id*/) { Assert (false, ExcInternalError()); @@ -1172,9 +1173,32 @@ estimate (const Mapping<1,spacedim> &mapping, const std::vector &component_mask_, const Function<1> *coefficient, const unsigned int, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id_, const unsigned int material_id) { + if (dynamic_cast*> + (&dof_handler.get_tria()) + != 0) + Assert ((subdomain_id_ == types::invalid_subdomain_id) + || + (subdomain_id_ == + dynamic_cast&> + (dof_handler.get_tria()).locally_owned_subdomain()), + ExcMessage ("For parallel distributed triangulations, the only " + "valid subdomain_id that can be passed here is the " + "one that corresponds to the locally owned subdomain id.")); + + const types::subdomain_id_t subdomain_id + = ((dynamic_cast*> + (&dof_handler.get_tria()) + != 0) + ? + dynamic_cast&> + (dof_handler.get_tria()).locally_owned_subdomain() + : + subdomain_id_); + + const unsigned int n_components = dof_handler.get_fe().n_components(); const unsigned int n_solution_vectors = solutions.size(); @@ -1273,7 +1297,7 @@ estimate (const Mapping<1,spacedim> &mapping, typename DH::active_cell_iterator cell = dof_handler.begin_active(); for (unsigned int cell_index=0; cell != dof_handler.end(); ++cell, ++cell_index) - if (((subdomain_id == numbers::invalid_unsigned_int) + if (((subdomain_id == types::invalid_subdomain_id) || (cell->subdomain_id() == subdomain_id)) && @@ -1416,7 +1440,7 @@ estimate (const Mapping &mapping, const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { // just pass on to the other function @@ -1439,7 +1463,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1462,7 +1486,7 @@ estimate (const Mapping &mapping, const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { // just pass on to the other function @@ -1485,7 +1509,7 @@ estimate (const DH &dof_handler, const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1510,9 +1534,32 @@ estimate (const Mapping &mapping, const std::vector &component_mask_, const Function *coefficients, const unsigned int , - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id_, const unsigned int material_id) { + if (dynamic_cast*> + (&dof_handler.get_tria()) + != 0) + Assert ((subdomain_id_ == types::invalid_subdomain_id) + || + (subdomain_id_ == + dynamic_cast&> + (dof_handler.get_tria()).locally_owned_subdomain()), + ExcMessage ("For parallel distributed triangulations, the only " + "valid subdomain_id that can be passed here is the " + "one that corresponds to the locally owned subdomain id.")); + + const types::subdomain_id_t subdomain_id + = ((dynamic_cast*> + (&dof_handler.get_tria()) + != 0) + ? + dynamic_cast&> + (dof_handler.get_tria()).locally_owned_subdomain() + : + subdomain_id_); + + const unsigned int n_components = dof_handler.get_fe().n_components(); // sanity checks @@ -1610,7 +1657,7 @@ estimate (const Mapping &mapping, for (typename DH::active_cell_iterator cell=dof_handler.begin_active(); cell!=dof_handler.end(); ++cell, ++present_cell) - if ( ((subdomain_id == numbers::invalid_unsigned_int) + if ( ((subdomain_id == types::invalid_subdomain_id) || (cell->subdomain_id() == subdomain_id)) && @@ -1660,7 +1707,7 @@ estimate (const Mapping &mapping, const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { // forward to the function with the QCollection @@ -1682,7 +1729,7 @@ void KellyErrorEstimator::estimate (const DH const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1703,7 +1750,7 @@ void KellyErrorEstimator::estimate (const DH const std::vector &component_mask, const Function *coefficients, const unsigned int n_threads, - const unsigned int subdomain_id, + const types::subdomain_id_t subdomain_id, const unsigned int material_id) { Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping")); @@ -1806,6 +1853,10 @@ INSTANTIATE(PETScWrappers::BlockVector,DoFHandler); INSTANTIATE(PETScWrappers::Vector,hp::DoFHandler); INSTANTIATE(PETScWrappers::BlockVector,hp::DoFHandler); + +INSTANTIATE(PETScWrappers::MPI::Vector,DoFHandler); +INSTANTIATE(PETScWrappers::MPI::BlockVector,DoFHandler); + #endif #ifdef DEAL_II_USE_TRILINOS diff --git a/deal.II/deal.II/source/numerics/solution_transfer.cc b/deal.II/deal.II/source/numerics/solution_transfer.cc index a9a64caa2b..ceddc927c3 100644 --- a/deal.II/deal.II/source/numerics/solution_transfer.cc +++ b/deal.II/deal.II/source/numerics/solution_transfer.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 by the deal.II authors +// Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -32,7 +32,8 @@ DEAL_II_NAMESPACE_OPEN template -SolutionTransfer::SolutionTransfer(const DH &dof): +SolutionTransfer::SolutionTransfer(const DH &dof) + : dof_handler(&dof, typeid(*this).name()), n_dofs_old(0), prepared_for(none) @@ -67,9 +68,9 @@ void SolutionTransfer::clear () template void SolutionTransfer::prepare_for_pure_refinement() -{ +{ Assert(prepared_for!=pure_refinement, ExcAlreadyPrepForRef()); - Assert(prepared_for!=coarsening_and_refinement, + Assert(prepared_for!=coarsening_and_refinement, ExcAlreadyPrepForCoarseAndRef()); clear(); @@ -84,7 +85,7 @@ void SolutionTransfer::prepare_for_pure_refinement() typename DH::active_cell_iterator cell = dof_handler->begin_active(), endc = dof_handler->end(); - for (unsigned int i=0; cell!=endc; ++cell, ++i) + for (unsigned int i=0; cell!=endc; ++cell, ++i) { indices_on_cell[i].resize(cell->get_fe().dofs_per_cell); // on each cell store the indices of the @@ -100,7 +101,7 @@ void SolutionTransfer::prepare_for_pure_refinement() } -template +template void SolutionTransfer::refine_interpolate(const VECTOR &in, VECTOR &out) const @@ -122,10 +123,10 @@ SolutionTransfer::refine_interpolate(const VECTOR &in, pointerstruct, cell_map_end=cell_map.end(); - for (; cell!=endc; ++cell) + for (; cell!=endc; ++cell) { pointerstruct=cell_map.find(std::make_pair(cell->level(),cell->index())); - + if (pointerstruct!=cell_map_end) // this cell was refined or not // touched at all, so we can get @@ -160,9 +161,9 @@ SolutionTransfer:: prepare_for_coarsening_and_refinement(const std::vector &all_in) { Assert(prepared_for!=pure_refinement, ExcAlreadyPrepForRef()); - Assert(!prepared_for!=coarsening_and_refinement, + Assert(!prepared_for!=coarsening_and_refinement, ExcAlreadyPrepForCoarseAndRef()); - + const unsigned int in_size=all_in.size(); Assert(in_size!=0, ExcNoInVectorsGiven()); @@ -185,7 +186,7 @@ prepare_for_coarsening_and_refinement(const std::vector &all_in) typename DH::active_cell_iterator act_cell = dof_handler->begin_active(), endc = dof_handler->end(); - for (; act_cell!=endc; ++act_cell) + for (; act_cell!=endc; ++act_cell) { if (act_cell->coarsen_flag_set()) ++n_cells_to_coarsen; @@ -201,7 +202,7 @@ prepare_for_coarsening_and_refinement(const std::vector &all_in) for (; cell!=endc; ++cell) if (!cell->active() && cell->child(0)->coarsen_flag_set()) ++n_coarsen_fathers; - + if (n_cells_to_coarsen) Assert(n_cells_to_coarsen>=2*n_coarsen_fathers, ExcInternalError()); @@ -211,18 +212,18 @@ prepare_for_coarsening_and_refinement(const std::vector &all_in) std::vector > (n_cells_to_stay_or_refine) .swap(indices_on_cell); - + std::vector > > (n_coarsen_fathers, std::vector > (in_size)) .swap(dof_values_on_cell); - + // we need counters for // the 'to_stay_or_refine' cells 'n_sr' and // the 'coarsen_fathers' cells 'n_cf', unsigned int n_sr=0, n_cf=0; - cell = dof_handler->begin(); - for (; cell!=endc; ++cell) + cell = dof_handler->begin(); + for (; cell!=endc; ++cell) { if (cell->active() && !cell->coarsen_flag_set()) { @@ -250,7 +251,7 @@ prepare_for_coarsening_and_refinement(const std::vector &all_in) std::vector >(in_size, Vector(dofs_per_cell)) .swap(dof_values_on_cell[n_cf]); - + for (unsigned int j=0; j &all_in, typename DH::cell_iterator cell = dof_handler->begin(), endc = dof_handler->end(); - for (; cell!=endc; ++cell) + for (; cell!=endc; ++cell) { pointerstruct=cell_map.find(std::make_pair(cell->level(),cell->index())); - + if (pointerstruct!=cell_map_end) { const std::vector * const indexptr @@ -320,7 +321,7 @@ interpolate (const std::vector &all_in, =pointerstruct->second.dof_values_ptr; const unsigned int dofs_per_cell=cell->get_fe().dofs_per_cell; - + // cell stayed or is // refined if (indexptr) diff --git a/deal.II/doc/development/Makefile b/deal.II/doc/development/Makefile index 9ce2b295f7..8fa9834712 100644 --- a/deal.II/doc/development/Makefile +++ b/deal.II/doc/development/Makefile @@ -64,7 +64,8 @@ makefiles.html: Makefile makefiles.1.html $D/common/Make.global_options makefile @echo '
  • INCLUDE=$(INCLUDE)' >> $@ @echo '
  • CXXFLAGS.g=$(CXXFLAGS.g)' >> $@ @echo '
  • CXXFLAGS.o=$(CXXFLAGS.o)' >> $@ - @echo '
  • CFLAGS=$(CFLAGS)' >> $@ + @echo '
  • CFLAGS.g=$(CFLAGS.g)' >> $@ + @echo '
  • CFLAGS.o=$(CFLAGS.o)' >> $@ @echo '
  • F77FLAGS.g=$(F77FLAGS.g)' >> $@ @echo '
  • F77FLAGS.o=$(F77FLAGS.o)' >> $@ @echo '
  • LDFLAGS=$(LDFLAGS)' >> $@ diff --git a/deal.II/doc/development/makefiles.1.html b/deal.II/doc/development/makefiles.1.html index ece06b10e5..3fc1566bb4 100644 --- a/deal.II/doc/development/makefiles.1.html +++ b/deal.II/doc/development/makefiles.1.html @@ -471,12 +471,16 @@

    -
    CCFLAGS
    +
    CCFLAGS.g

    - C compiler flags. Since we only compiler C code for - third-party libraries for which we assume that they and our - interfaces to them are bug free, these flags are always for - optimized mode. + C compiler flags for debug mode. +

    +
    + + +
    CCFLAGS.o
    +

    + C compiler flags for optimized mode.

    diff --git a/deal.II/doc/doxygen/headers/distributed.h b/deal.II/doc/doxygen/headers/distributed.h new file mode 100644 index 0000000000..2c5df2d112 --- /dev/null +++ b/deal.II/doc/doxygen/headers/distributed.h @@ -0,0 +1,364 @@ +//------------------------------------------------------------------------- +// $Id$ +// Version: $Name$ +// +// Copyright (C) 2009, 2010 by the deal.II authors +// +// This file is subject to QPL and may not be distributed +// without copyright and license information. Please refer +// to the file deal.II/doc/license.html for the text and +// further information on this license. +// +//------------------------------------------------------------------------- + +/** + * @defgroup distributed Parallel computing with multiple processors using distributed memory + * @ingroup Parallel + * + * @brief A module discussing the use of parallelism on distributed memory + * clusters. + * + *

    Overview

    + * + * deal.II can use multiple machine connected via MPI to parallelize + * computations, in addition to the parallelization within a shared + * memory machine discussed in the @ref threads module. There are + * essentially two ways to utilize multiple machines: + * + * - Each machine keeps the entire mesh and DoF handler locally, but + * only a share of the global matrix, sparsity pattern, and solution + * vector is stored on each machine. + * - The mesh and DoFhandler are also distributed, i.e. each processor + * stores only a share of the cells and degrees of freedom. No + * processor has knowledge of the entire mesh, matrix, or solution, + * and in fact problems solved in this mode are usually so large + * (say, 100s of millions to billions of degrees of freedom) that no + * processor can or should store even a single solution vector. + * + * The first of these two options is relatively straightforward + * because most of the things one wants to do in a finite element + * program still work in essentially the same way, and handling + * distributed matrices, vectors, and linear solvers is something for + * which good external libraries such as @ref SoftwareTrilinos or @ref + * SoftwarePETSc exist that can make things look almost exactly the + * same as they would if everything was available locally. The use of + * this mode of parallelization is explained in the tutorial programs + * step-17, and step-18 and will not be discussed here in more detail. + * + * The use of truly distributed meshes is somewhat more complex because it + * changes or makes impossible some of the things that can otherwise be done + * with deal.II triangulations, DoF handlers, etc. This module documents these + * issues with a vantage point at 50,000 ft above ground without going into + * too many details. All the algorithms described below are implement in + * classes and functions in namespace parallel::distributed. + * + * + *

    Other resources

    + * + * A complete discussion of the algorithms used in this namespace, as well as + * a thorough description of many of the terms used here, can be found in the + * @ref distributed_paper "Distributed Computing paper". In particular, the + * paper shows that the methods discussed in this module scale to thousands of + * processors and well over a billion degrees of freedom (they may scale to + * even bigger problems but at the time of writing this, we do not have + * solvers that are capable of more than $2^{31}$ degrees of freedom due to + * the use of signed int as index). The paper also gives a + * concise definition of many of the terms that are used here and in other + * places of the library related to distributed computing. The step-40 + * tutorial program shows an application of the classes and methods of this + * namespace to the Laplace equation, while step-32 extends the step-31 + * program to massively parallel computations and thereby explains the use of + * the topic discussed here to more complicated applications. + * + * + *

    Distributed triangulations

    + * + * In %parallel %distributed mode, objects of type + * parallel::distributed::Triangulation on each processor only store + * a subset of cells. In particular, the global mesh can be thought of + * as decomposed so that each MPI process "owns" a number of + * cells. The mesh each process then stores locally consists of + * exactly those cells that it owns, as well as one layer of @ref + * GlossGhostCell "ghost cells" around the ones it locally owns, and a + * number of cells we call @ref GlossArtificialCell "artificial". The + * latter are cells that ensure that each processor has a mesh that + * has all the coarse level cells and that respects the invariant that + * neighboring cells can not differ by more than one level of + * refinement. The following pictures show such a mesh, %distributed + * across four processors, and the collection of cells each of these + * processors stores locally: + * + * + * + * + * + * + * + * + * + * + *
    @image html distributed_mesh_0.png @image html distributed_mesh_1.png
    @image html distributed_mesh_2.png @image html distributed_mesh_3.png
    + * + * The cells are colored based on the @ref GlossSubdomainId "subdomain id", + * which identifies which processor owns a cell: turquoise for + * processor 0, green for processor 1, yellow for processor 2, and red + * for processor 3. As can be seen, each process has one layer of + * ghost cells around its own cells, which are correctly colored by + * the subdomain id that identifies the processor that owns each of + * these cells. Note also how each processor stores a number of + * artificial cells, indicated in blue, that only exist to ensure that + * each processor knows about all coarse grid cells and that the + * meshes have the 2:1 refinement property; however, in the area + * occupied by these artificial cells, a processor has no knowledge + * how refined the mesh there really is, as these are areas that are + * owned by other processors. As a consequence, all algorithms we will + * develop can only run over the locally owned cells and if necessary + * the ghost cells; trying to access data on any of the artificial + * cells is most likely an error. Note that we can determine whether + * we own a cell by testing that cell-@>subdomain_id() == + * triangulation.locally_owned_subdomain(). + * + * The "real" mesh one has to think of here is the one that would + * result from forming the union of cells each of the processes own, + * i.e. from the overlap of the turquoise, green, yellow and red + * areas, disregarding the blue areas. + * + * + *

    Distributed degree of freedom handler

    + * + * The DoFHandler class builds on the Triangulation class, but it can + * detect whenever we actually use an object of type + * parallel::distributed::Triangulation as triangulation. In that + * case, it assigns global %numbers for all degrees of freedom that + * exist, given a finite element, on the global mesh, but each + * processor will only know about those that are defined on locally + * relevant cells (i.e. cells either locally owned or that are ghost + * cells). Internally, the algorithm essentially works by just looping + * over all cells we own locally and assigning DoF indices to the + * degrees of freedom defined on them and, in the case of degrees of + * freedom at the interface between subdomains owned by different + * processors, that are not owned by the neighboring processor. All + * processors then exchange how many degrees of freedom they locally + * own and shift their own indices in such a way that every degree of + * freedom on all subdomains are uniquely identified by an index + * between zero and DoFHandler::n_dofs() (this function returns the + * global number of degrees of freedom, accumulated over all + * processors). Note that after this step, the degrees of freedom + * owned by each process form a contiguous range that can, for + * example, be obtained by the contiguous index set returned by + * DoFHandler::locally_owned_dofs(). After + * assigning unique indices to all degrees of freedom, the + * DoFHandler::distribute_dofs() function then + * loops over all ghost cells and communicates with neighboring + * processors to ensure that the global indices of degrees of freedom + * on these ghost cells match the ones that the neighbor has assigned + * to them. + * + * Through this scheme, we can make sure that each cell we locally own + * as well as all the ghost cells can be asked to yield the globally + * correct indices for the degrees of freedom defined on + * them. However, asking for degrees of freedom on artificial cells is + * likely going to lead to nothing good, as no information is + * available for these cells (in fact, it isn't even known whether + * these cells are active on the global mesh, or are further refined). + * + * As usual, degrees of freedom can be renumbered after being enumerated, + * using the functions in namespace DoFRenumbering. + * + * + *

    Linear systems for %distributed computations

    + * + * One thing one learns very quickly when working with very large + * numbers of processors is that one can not store information about + * every degree of freedom on each processor, even if this information + * is "this degree of freedom doesn't live here". An example for this + * is that we can create an object for a (compressed) sparsity pattern + * that has DoFHandler::n_dofs() rows, + * but for which we fill only those rows that correspond to the + * DoFHandler::n_locally_owned_dofs() locally + * owned degrees of freedom. The reason is simple: for the sake of + * example, let's assume we have 1 billion degrees of freedom + * distributed across 100 processors; if we even only hold 16 bytes + * per line in this sparsity pattern (whether we own the corresponding + * DoF or not), we'll need 16 GB for this object even if every single + * line is empty. Of course, only 10 million lines will be non-empty, + * for which we need 160 MB plus whatever is necessary to store the + * actual column indices of nonzero entries. Let's say we have a + * moderately complex problem with 50 entries per row, for each of + * which we store the column index worth 4 bytes, then we'll need 216 + * bytes for each of the 10 million lines that correspond to the + * degrees of freedom we own, for a total of 2.16 GB. And we'll need + * 16 bytes for each of the 990 million lines that we don't own, for a + * total of 15.840 GB. It is clear that this ratio doesn't become any + * better if we go to even higher %numbers of processors. + * + * The solution to this problem is to really only use any memory at + * all for those parts of the linear system that we own, or need for + * some other reason. For all other parts, we must know that they + * exist, but we can not set up any part of our data structure. To + * this end, there exists a class called IndexSet that denotes a set + * of indices which we care for, and for which we may have to allocate + * memory. The data structures for sparsity patterns, constraint + * matrices, matrices and vector can be initialized with these + * IndexSet objects to really only care for those rows or entries that + * correspond to indices in the index set, and not care about all + * others. These objects will then ask how many indices exist in the + * set, allocate memory for each one of them (e.g. initialize the data + * structures for a line of a sparsity pattern), and when you want to + * access data for global degree of freedom i you will be + * redirected to the result of calling IndexSet::index_within_set() + * with index i instead. Accessing data for elements + * i for which IndexSet::is_element() is false will yield + * an error. + * + * The remaining question is how to identify the set of indices that + * correspond to degrees of freedom we need to worry about on each + * processor. To this end, you can use the + * DoFTools::extract_locally_owned_dofs() function to get at all the + * indices a processor owns. Note that this is a subset of the degrees + * of freedom that are defined on the locally owned cells (since some + * of the degrees of freedom at the interface between two different + * subdomains may be owned by the neighbor). This set of degrees of + * freedom defined on cells we own can be obtained using the function + * DoFTools::extract_locally_active_dofs(). Finally, one + * sometimes needs the set of all degrees of freedom on the locally + * owned subdomain as well as the adjacent ghost cells. This + * information is provided by the + * DoFTools::extract_locally_relevant_dofs() function. + * + * + *
    Sparsity patterns
    + * + * At the time of writing this, the only class equipped to deal with the + * situation just explained is CompressedSimpleSparsityPattern. A version of + * the function CompressedSimpleSparsityPattern::reinit() exists that takes an + * IndexSet argument that indicates which lines of the sparsity pattern to + * allocate memory for. In other words, it is safe to create such an object + * that will report as its size 1 billion, but in fact only stores only as + * many rows as the index set has elements. You can then use the usual + * function DoFTools::make_sparsity_pattern to build the sparsity pattern that + * results from assembling on the locally owned portion of the mesh. The + * resulting object can be used to initialize a PETSc or Trilinos matrix which + * support very large object sizes through completely distributed storage. The + * matrix can then be assembled by only looping over those cells owned by the + * current processor. + * + * The only thing to pay attention to is for which degrees of freedom the + * sparsity needs to store entries. These are, in essence, the ones we could + * possibly store values to in the matrix upon assembly. It is clear that + * these are certainly the locally active degrees of freedom (which live on + * the cells we locally own) but through constraints, it may also be possible + * to write to entries that are located on ghost cells. Consequently, you need + * to pass the index set that results from + * DoFTools::extract_locally_relevant_dofs() upon initializing the sparsity + * pattern. + * + * + *

    Constraints on degrees of freedom

    + * + * When creating the sparsity pattern as well as when assembling the linear + * system, we need to know about constraints on degrees of freedom, for + * example resulting from hanging nodes or boundary conditions. Like the + * CompressedSimpleSparsityPattern class, the ConstraintMatrix can also take + * an IndexSet upon construction that indicates for which of the possibly very + * large number of degrees of freedom it should actually store + * constraints. Unlike for the sparsity pattern, these are now only those + * degrees of freedom which we work on locally when assembling, namely those + * returned by DoFTools::extract_locally_active_dofs() (a superset of the + * locally owned ones). + * + * There are, however, situations where more complicated constraints appear in + * finite element programs. An example is in $hp$ adaptive computations where + * degrees of freedom can be constrained against other degrees of freedom that + * are themselves constrained. In a case like this, in order to fully resolve + * this chain of constraints, it may not be sufficient to only store + * constraints on locally active degrees of freedom but one may also need to + * have constraints available on locally relevant ones. In that case, the + * ConstraintMatrix object needs to be initialized with the IndexSet produced + * by DoFTools::extract_locally_relevant_dofs() . + * + * In general, your program will continue to do something if you happen to not + * store all necessary constraints on each processor: you will just generate + * wrong matrix entries, but the program will not abort. This is opposed to + * the situation of the sparsity pattern: there, if the IndexSet passed to the + * CompressedSimpleSparsityPattern indicates that it should store too few rows + * of the matrix, the program will either abort when you attempt to write into + * matrix entries that do not exist or the matrix class will silently allocate + * more memory to accomodate them. As a consequence, it is useful to err on + * the side of caution when indicating which constraints to store and use the + * result of DoFTools::extract_locally_relevant_dofs() rather than + * DoFTools::extract_locally_active_dofs() . This is also affordable since the + * set of locally relevant degrees of freedom is only marginally larger than + * the set of locally active degrees of freedom. We choose this strategy in + * both step-32 and step-40. + * + * + *

    Postprocessing

    + * + * Like everything else, you can only do postprocessing on cells a + * local processor owns. The DataOut and KellyErrorEstimator classes + * do this automatically: they only operate on locally owned cells + * without the need to do anything in particular. At least for large + * computations, there is also no way to merge the results of all + * these local computations on a single machine, i.e. each processor + * has to be self-sufficient. For example, each processor has to + * generate its own parallel output files that have to be visualizated + * by a program that can deal with multiple input files rather than + * merging the results of calling DataOut to a single processor before + * generating a single output file. The latter can be achieved, for + * example, using the DataOutBase::write_vtu() and + * DataOutBase::write_pvtu_record() functions. + * + * These same considerations hold for all other postprocessing actions + * as well: while it is, for example, possible to compute a global + * energy dissipation rate by doing the computations locally and + * accumulating the resulting single number processor to a single + * number for the entire communication, it is in general not possible + * to do the same if the volume of data produced by every processor is + * significant. + * + * There is one particular consideration for postprocessing, however: whatever + * you do on each cell a processor owns, you need access to at least all those + * values of the solution vector that are active on these cells (i.e. to the + * set of all locally active degrees of freedom, in the language of the + * @ref distributed_paper "Distributed Computing paper"), which is a superset + * of the degrees of freedom this processor actually owns (because it may not + * own all degrees of freedom on the interface between its own cells and those + * cells owned by other processors). Sometimes, however, you need even more + * information: for example, to compute the KellyErrorIndicator results, one + * needs to evaluate the gradient at the interface on the current as well as + * its neighbor cell; the latter may be owned by another processor, so we need + * those degrees of freedom as well. In general, therefore, one needs access + * to the solution values for all degrees of freedom that are locally + * relevant. On the other hand, both of the packages we can use for + * parallel linear algebra (PETSc and Trilinos) subdivide vectors into chunks + * each processor owns and chunks stored on other processors. To postprocess + * stuff therefore means that we have to tell PETSc or Trilinos that it should + * also import ghost elements, i.e. additional vector elements of the + * solution vector other than the ones we store locally. Both the + * PETScWrappers::MPI::Vector and TrilinosWrappers::MPI::Vector class support + * specifying this information (see step-40 and step-32, respectively) through + * the PETScWrappers::MPI::Vector::update_ghost_values() function or, in the + * case of Trilinos, construction of a vector with an the locally relevant + * degrees of freedom index set. + */ + + + +namespace parallel +{ + /** + * A namespace for class and + * functions that support %parallel + * computing on %distributed memory + * machines. See the @ref + * distributed module for an + * overview of the facilities this + * namespace offers. + * + * @ingroup distributed + */ + namespace distributed + { + } +} diff --git a/deal.II/doc/doxygen/headers/glossary.h b/deal.II/doc/doxygen/headers/glossary.h index d1a395fae5..c349e47699 100644 --- a/deal.II/doc/doxygen/headers/glossary.h +++ b/deal.II/doc/doxygen/headers/glossary.h @@ -426,13 +426,14 @@ *
    The "distributed computing paper" is a paper by W. Bangerth, * C. Burstedde, T. Heister and M. Kronbichler titled "Algorithms and Data * Structures for Massively Parallel Generic Finite Element Codes" that - * described the implementation of parallel distributed computing in deal.II, + * describes the implementation of parallel distributed computing in deal.II, * i.e. computations where not only the linear system is split onto different * machines as in, for example, step-18, but also the Triangulation and * DoFHandler objects. In essence, it is a guide to the parallel::distributed * namespace. * - * The paper is currently in preparation. + * The paper is submitted to the ACM Transactions on Mathematical Software + * (ACM TOMS). *
    * * @@ -608,21 +609,21 @@ Article{JK10, * polynomials. Let us give some examples: * * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * *
    ElementFunction spaceNode values
    FE_Q, FE_DGQQkvalues in support points
    FE_DGPPkmoments with respect to Legendre polynomials
    FE_RaviartThomas (2d)Qk+1,k x Qk,k+1moments on edges and in the interior
    FE_RaviartThomasNodalQk+1,k x Qk,k+1Gauss points on edges(faces) and anisotropic Gauss points in the interior
    ElementFunction spaceNode values
    FE_Q, FE_DGQQkvalues in support points
    FE_DGPPkmoments with respect to Legendre polynomials
    FE_RaviartThomas (2d)Qk+1,k x Qk,k+1moments on edges and in the interior
    FE_RaviartThomasNodalQk+1,k x Qk,k+1Gauss points on edges(faces) and anisotropic Gauss points in the interior
    * *
    @anchor GlossPrimitive Primitive finite @@ -633,7 +634,7 @@ Article{JK10, * a vector-valued element has exactly one nonzero component if an element is * primitive. This includes, in particular, all scalar elements as well as * vector-valued elements assembled via the FESystem class from other - * primitive (for example scalar) elements as shown in step-8, + * primitive (for example scalar) elements as shown in step-8, * step-29, step-22 and several others. On the other hand, * the FE_RaviartThomas class used in step-20 and step-21, or the FE_Nedelec * class provide non-primitive finite elements because there, each @@ -660,16 +661,15 @@ Article{JK10, * cell is associated with. * * For programs that are parallelized based on MPI but where each processor - * stores the entire triangulation (as in, for example, step-18 - * or step-32, subdomain ids are assigned to cells by + * stores the entire triangulation (as in, for example, step-18, but not in + * step-32), subdomain ids are assigned to cells by * partitioning a mesh, and each MPI process then only works on those cells it - * "owns", i.e. that belong to a subdomain that it is associated with + * "owns", i.e. that belong to a subdomain that the processor is associated with * (traditionally, this is the case for the subdomain id whose numerical value * coincides with the rank of the MPI process within the MPI * communicator). Partitioning is typically done using the * GridTools::partition() function, but any other method can also be used to - * do this though most simple ideas will likely lead to less well balanced - * numbers of degrees of freedom on the various subdomains. + * do this. * * On the other hand, for programs that are parallelized using MPI but * where meshes are held distributed across several processors using diff --git a/deal.II/doc/doxygen/headers/parallel.h b/deal.II/doc/doxygen/headers/parallel.h index 7189a61040..b61f9f58ea 100644 --- a/deal.II/doc/doxygen/headers/parallel.h +++ b/deal.II/doc/doxygen/headers/parallel.h @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2009, 2010 by the deal.II authors +// Copyright (C) 2009 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -17,9 +17,7 @@ * @brief A module discussing the use of multiple processor. * * This module contains information on %parallel computing. It is - * subdivided into parts on @ref threads and on distributed - * computing. The latter part will become available in a future - * public release. + * subdivided into parts on @ref threads and on @ref distributed. */ @@ -27,7 +25,8 @@ * A namespace in which we define classes and algorithms that deal * with running in %parallel on shared memory machines when deal.II is * configured to use multiple threads (see @ref threads), as well as - * running things in %parallel on %distributed memory machines. + * running things in %parallel on %distributed memory machines (see @ref + * distributed). * * @ingroup threads * @author Wolfgang Bangerth, 2008, 2009 diff --git a/deal.II/doc/doxygen/images/distributed_mesh_0.png b/deal.II/doc/doxygen/images/distributed_mesh_0.png new file mode 100644 index 0000000000..0975c9a7c1 Binary files /dev/null and b/deal.II/doc/doxygen/images/distributed_mesh_0.png differ diff --git a/deal.II/doc/doxygen/images/distributed_mesh_1.png b/deal.II/doc/doxygen/images/distributed_mesh_1.png new file mode 100644 index 0000000000..12864ae955 Binary files /dev/null and b/deal.II/doc/doxygen/images/distributed_mesh_1.png differ diff --git a/deal.II/doc/doxygen/images/distributed_mesh_2.png b/deal.II/doc/doxygen/images/distributed_mesh_2.png new file mode 100644 index 0000000000..16db6f6380 Binary files /dev/null and b/deal.II/doc/doxygen/images/distributed_mesh_2.png differ diff --git a/deal.II/doc/doxygen/images/distributed_mesh_3.png b/deal.II/doc/doxygen/images/distributed_mesh_3.png new file mode 100644 index 0000000000..9b97ad2d83 Binary files /dev/null and b/deal.II/doc/doxygen/images/distributed_mesh_3.png differ diff --git a/deal.II/doc/doxygen/tutorial/Makefile b/deal.II/doc/doxygen/tutorial/Makefile index a2641da2bf..64c314d503 100644 --- a/deal.II/doc/doxygen/tutorial/Makefile +++ b/deal.II/doc/doxygen/tutorial/Makefile @@ -44,7 +44,7 @@ all: $(example-doxygen) toc.html # # make steps.png # -# by hand. This way, we avoid testing for neato during +# by hand. This way, we avoid testing for dot during # configuration. The map only needs to be generated after changes in # the structure of the tutorials. toc.html: steps.cmapx toc.html.in @@ -52,13 +52,13 @@ toc.html: steps.cmapx toc.html.in @perl make_toc.pl > toc.html # generate dot file -steps.dot: steps.pl $D/examples/*/doc/tooltip +steps.dot: steps.pl $D/examples/*/doc/tooltip $D/examples/*/doc/builds-on @echo ================== Making $@ @perl $< > $@ steps.png: steps.dot @echo ================== Making $@ - @neato -Tpng -Tcmapx -O $< + @dot -Tpng -Tcmapx -O $< @mv steps.dot.png steps.png @mv steps.dot.cmapx steps.cmapx diff --git a/deal.II/doc/doxygen/tutorial/steps.cmapx b/deal.II/doc/doxygen/tutorial/steps.cmapx index 45c6f8238b..56f83e58d0 100644 --- a/deal.II/doc/doxygen/tutorial/steps.cmapx +++ b/deal.II/doc/doxygen/tutorial/steps.cmapx @@ -1,38 +1,40 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deal.II/doc/doxygen/tutorial/steps.pl b/deal.II/doc/doxygen/tutorial/steps.pl index 3b59d4290c..b5701aa516 100644 --- a/deal.II/doc/doxygen/tutorial/steps.pl +++ b/deal.II/doc/doxygen/tutorial/steps.pl @@ -11,18 +11,18 @@ use strict; my @steps = (1,2,3,4,5,6,7,8,9, 10,11,12,13,14,15,16,17,18,19, 20,21,22,23,24,25, 27,28,29, - 30,31, 33,34,35,36, 39, - 45); + 30,31,32,33,34,35,36, 39, + 40,45); ; # List of additional node attributes to highlight purpose and state of the example my %style = ( - "basic" => ',height=.7,width=.7,shape="octagon",fillcolor="green"', - "techniques" => ',fillcolor="orange"', - "fluids" => ',fillcolor="yellow"', - "solids" => ',fillcolor="lightblue"', - "time dependent" => ',fillcolor="blue"', - "unfinished" => ',style="dashed"' + "basic" => ',height=.8,width=.8,shape="octagon",fillcolor="green"', + "techniques" => ',height=.35,width=.35,fillcolor="orange"', + "fluids" => ',height=.25,width=.25,fillcolor="yellow"', + "solids" => ',height=.25,width=.25,fillcolor="lightblue"', + "time dependent" => ',height=.25,width=.25,fillcolor="blue"', + "unfinished" => ',height=.25,width=.25,style="dashed"' ); diff --git a/deal.II/doc/doxygen/tutorial/steps.png b/deal.II/doc/doxygen/tutorial/steps.png index 0894d21d15..44435ded1a 100644 Binary files a/deal.II/doc/doxygen/tutorial/steps.png and b/deal.II/doc/doxygen/tutorial/steps.png differ diff --git a/deal.II/doc/external-libs/p4est-setup.sh b/deal.II/doc/external-libs/p4est-setup.sh new file mode 100755 index 0000000000..69b856284f --- /dev/null +++ b/deal.II/doc/external-libs/p4est-setup.sh @@ -0,0 +1,110 @@ +#! /bin/bash + +# This program comes with ABSOLUTELY NO WARRANTY. + +# unpack under current directory +UNPACK=`pwd` +# choose names for fast and debug compilation directories +BUILD_FAST="$UNPACK/p4est-build/FAST" +BUILD_DEBUG="$UNPACK/p4est-build/DEBUG" + +function busage() { + echo "Usage: `basename $0` " +} +function bdie () { + echo "Error: $@" + exit 1 +} + +if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_FAST" ; then + export CFLAGS_FAST="-O2" +else + export CFLAGS_FAST="$CFLAGS $P4EST_CFLAGS_FAST" +fi +echo "CFLAGS_FAST: $CFLAGS_FAST" +if test -z "$CFLAGS" -a -z "$P4EST_CFLAGS_DEBUG" ; then + export CFLAGS_DEBUG="-O0 -g" +else + export CFLAGS_DEBUG="$CFLAGS $P4EST_CFLAGS_DEBUG" +fi +echo "CFLAGS_DEBUG: $CFLAGS_DEBUG" + +TGZ="$1"; shift +if test ! -f "$TGZ" ; then + busage + bdie "File not found" +fi +if ! (echo "$TGZ" | grep -q 'p4est.*.tar.gz') ; then + busage + bdie "File name mismatch" +fi + +# choose names for fast and debug installation directories +INSTALL_DIR=$1 +shift +if test -z "$INSTALL_DIR" ; then + INSTALL_FAST="$UNPACK/p4est-install/FAST" + INSTALL_DEBUG="$UNPACK/p4est-install/DEBUG" +else + INSTALL_FAST="$INSTALL_DIR/FAST" + INSTALL_DEBUG="$INSTALL_DIR/DEBUG" +fi + +echo +echo "This script tries to unpack, configure and build the p4est library." +echo "Build FAST: $BUILD_FAST" +echo "Build DEBUG: $BUILD_DEBUG" +echo "Install FAST: $INSTALL_FAST" +echo "Install DEBUG: $INSTALL_DEBUG" +echo "Checking environment: CFLAGS P4EST_CFLAGS_FAST P4EST_CFLAGS_DEBUG" + + +if test -d $UNPACK/p4est-build ; then + rm -rf $UNPACK/p4est-build +fi + +DIR=`echo "$TGZ" | sed 's/\(p4est.*\).tar.gz/\1/'` +DIR=`basename $DIR` +echo "Unpack directory: $UNPACK/$DIR" +if test -d "$UNPACK/$DIR" ; then + echo \ + "Directory found (remove it and also the build directories" \ + "to start over)" +else + echo -n "Unpacking... " + tar -xvz -f "$TGZ" -C "$UNPACK" >/dev/null + echo "done" +fi +test -f "$UNPACK/$DIR/src/p4est.h" || bdie "Main header file missing" +test -f "$UNPACK/$DIR/configure" || bdie "Configure script missing" + +echo +echo "See output in files .../config.output and .../make.output" +echo "Build FAST version in $BUILD_FAST" +mkdir -p "$BUILD_FAST" +cd "$BUILD_FAST" +("$UNPACK/$DIR/configure" --enable-mpi --enable-shared --disable-vtk-binary --without-blas \ + --prefix="$INSTALL_FAST" CFLAGS="$CFLAGS_FAST" \ + "$@" || bdie "Error in configure" ) | tee config.output +(make -C sc -j 8 || bdie "Error in make sc") | tee make.output +(make src/libp4est.la -j 8 \ + || bdie "Error in make p4est") | tee -a make.output +(make install || bdie "Error in make install") | tee -a make.output +echo "FAST version installed in $INSTALL_FAST" + +echo +echo "Build DEBUG version in $BUILD_DEBUG" +mkdir -p "$BUILD_DEBUG" +cd "$BUILD_DEBUG" +if test -z "$CFLAGS" ; then + export CFLAGS="-g -O0" +fi +("$UNPACK/$DIR/configure" --enable-mpi --enable-shared --disable-vtk-binary --without-blas --enable-debug \ + --prefix="$INSTALL_DEBUG" CFLAGS="$CFLAGS_DEBUG" \ + "$@" || bdie "Error in configure") | tee config.output +(make -C sc -j 8 || bdie "Error in make sc") | tee make.output +(make src/libp4est.la -j 8 \ + || bdie "Error in make p4est") | tee -a make.output +(make install || bdie "Error in make install") | tee -a make.output +echo "DEBUG version installed in $INSTALL_DEBUG" +echo diff --git a/deal.II/doc/external-libs/p4est.html b/deal.II/doc/external-libs/p4est.html new file mode 100644 index 0000000000..a18ef34ef1 --- /dev/null +++ b/deal.II/doc/external-libs/p4est.html @@ -0,0 +1,62 @@ + + + + + The deal.II Readme + + + + + + + +

    Using and installing instructions for the p4est library

    + +

    + p4est is a + library that manages meshes that are distributed across multiple + processors. It forms the basis of deal.II's implementation of + finite element solvers that can use meshes that are too large to + be held on each processor individually. +

    + +

    + You need to install p4est before deal.II. To do so, you can + download it + from here, copy it to a fresh directory + into which you should also copy + the p4est-setup.sh script. Then + call the script as follows: + +

    +	  ./p4est-setup.sh p4est-x-y-z.tar.gz /path/to/installation
    +	
    + + where p4est-x-y-z.tar.gz is the name of the p4est + distribution file, and /path/to/installation is a + directory into which you want to install p4est. +

    + +

    + After this, you need to configure and build deal.II using a line + like + +

    +	  ./configure --with-p4est=/path/to/installation --enable-mpi
    +	  make all
    +	
    + + Obviously, you can also add additional flags + to ./configure as described in the + general ReadMe file. +

    + +
    + +
    + The deal.II mailing list +
    + + diff --git a/deal.II/doc/readme.html b/deal.II/doc/readme.html index 5718222251..7d00c4e0a8 100644 --- a/deal.II/doc/readme.html +++ b/deal.II/doc/readme.html @@ -471,30 +471,34 @@ processor cores. If this is not desired, use --disable-threads. +
  • +

    + --enable-mpi: If given this + flag, ./configure chooses mpiCC + and mpicc as the C++ and C compilers, respectively, + unless the $CXX and $CC environment + variables have specifically been set to something else. If these + compilers exist and indeed support MPI, then this also switches + on support for MPI in the library. +

    +
  • --with-cpu=...: Enable specific optimization flags for a particular processor. Programs compiled with these options - might not execute on an other system, but will be faster by up to - 30 percent on the CPU selected. + might not execute on an other system, but may be faster on the + particular CPU selected.

    For a complete list of supported values of this switch, you may take a look at the file aclocal.m4 in the top-level directory. However, the most commonly used value - is --with-cpu=native indicating that the compiler + is --with-cpu=native, indicating that the compiler should determine the CPU we are running on by itself and optimize for it.

    -

    - Optimizations currently only work in conjunction with gcc. Without the - option --with-cpu=xxx, no cpu specific optimization is applied. - Furthermore, CPU specific optimization is switched on only for the compilation - of the optimized library versions. -

    -
  • --with-doxygen=...: Select the specified executable @@ -657,6 +661,21 @@

    + +
    p4est
    +
    +

    + p4est is a library that deal.II uses to + distribute very large meshes across multiple processors (think + meshes with a billion cells on 10,000 processors). Using and + installing p4est is discussed here. + To configure deal.II with p4est, you will + need to use the --with-path=/path/to/path + switch to the ./configure script. +

    +
    +
    BLAS, LAPACK
    @@ -730,10 +749,10 @@
    MUltifrontal Massively Parallel sparse direct Solver (MUMPS)
    - It is possible to make use some subroutines that make use of the + It is possible to make use some subroutines that make use of the MUltifrontal Massively Parallel sparse direct Solver - (MUMPS). For a detailed description of how to compile MUMPS (and + target="_top">MUltifrontal Massively Parallel sparse direct Solver + (MUMPS). For a detailed description of how to compile MUMPS (and some dependencies) and linking with deal.II, see this page.
    @@ -754,7 +773,7 @@
    ARPACK
    - There is a wrapper for the + There is a wrapper for the ARPACK library, which has its own license; if you want to use it with deal.II, please read it and make sure that you agree with it. diff --git a/deal.II/examples/step-18/doc/builds-on b/deal.II/examples/step-18/doc/builds-on index e820585ca6..1b9190dc94 100644 --- a/deal.II/examples/step-18/doc/builds-on +++ b/deal.II/examples/step-18/doc/builds-on @@ -1 +1 @@ -step-17 +step-17 step-19 diff --git a/deal.II/examples/step-32/Makefile b/deal.II/examples/step-32/Makefile index 770b78a94b..fce563c47d 100644 --- a/deal.II/examples/step-32/Makefile +++ b/deal.II/examples/step-32/Makefile @@ -30,7 +30,7 @@ D = ../../ # shall be deleted when calling `make clean'. Object and backup files, # executables and the like are removed anyway. Here, we give a list of # files in the various output formats that deal.II supports. -clean-up-files = *gmv *gnuplot *gpl *eps *pov *vtk +clean-up-files = *gmv *gnuplot *gpl *eps *pov *vtk *d2 diff --git a/deal.II/examples/step-32/doc/builds-on b/deal.II/examples/step-32/doc/builds-on index 11e000dda8..63490605a0 100644 --- a/deal.II/examples/step-32/doc/builds-on +++ b/deal.II/examples/step-32/doc/builds-on @@ -1 +1 @@ -step-31 step-17 +step-31 step-17 step-40 diff --git a/deal.II/examples/step-32/step-32.cc b/deal.II/examples/step-32/step-32.cc index 9ca65c24a7..6eaac2ef88 100644 --- a/deal.II/examples/step-32/step-32.cc +++ b/deal.II/examples/step-32/step-32.cc @@ -1,6 +1,7 @@ //TODO: - adjust stopping criteria for solvers // - better refinement at the start? // - check solver stability +// - Q2 Mapping useful? /* $Id$ */ @@ -31,12 +32,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -74,11 +77,86 @@ // matrices. #include +#include +#include +#include +#include + +#include "timeblock.h" + + + + #include + #include + #include + #include + +class BadConversion : public std::runtime_error { + public: + BadConversion(std::string const& s) + : std::runtime_error(s) + { } +}; + +inline std::string stringify(double x) +{ + std::ostringstream o; + if (!(o << x)) + throw BadConversion("stringify(double)"); + return o.str(); +} + + + +#define CONF_LOG_SOLVER +//#define CONF_COMPARE_SOLVER +//#define CONF_BENCHMARK - // Next, we import all deal.II - // names into global namespace: using namespace dealii; +void print_it(Utilities::System::MinMaxAvg & result) +{ + std::cout// << "sum: " << result.sum + << " avg: " << (long)result.avg/1024 + << " min: " << (long)result.min/1024 << " @" << result.min_index + << " max: " << (long)result.max/1024 << " @" << result.max_index + << std::endl; +} + +void print_memory_stats() +{ + int myid = Utilities::System::get_this_mpi_process(MPI_COMM_WORLD); + Utilities::System::MemoryStats stats; + Utilities::System::get_memory_stats(stats); + Utilities::System::MinMaxAvg r; + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmPeak, r); + if (myid==0) + { + std::cout << "MEM: VmPeak: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmSize, r); + if (myid==0) + { + std::cout << "MEM: VmSize: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmHWM, r); + if (myid==0) + { + std::cout << "MEM: VmHWM: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmRSS, r); + if (myid==0) + { + std::cout << "MEM: VmRSS: "; + print_it(r); + } +} + + +static int out_index=0; // @sect3{Equation data} // In the following namespace, we define the @@ -145,8 +223,17 @@ namespace EquationData const double h = R1-R0; const double s = (r-R0)/h; +// see http://www.wolframalpha.com/input/?i=plot+(sqrt(x^2%2By^2)*0.95%2B0.05*sin(6*atan2(x,y))),+x%3D-1+to+1,+y%3D-1+to+1 - return T1+(T0-T1)*((1-s)*(1-s)); + double s_mod = s*0.95 + 0.05*sin(6.0*atan2(p(0),p(1))); +//alternative: http://www.wolframalpha.com/input/?i=plot+atan((sqrt(x^2%2By^2)*0.95%2B0.05*sin(6*atan2(x,y))-0.5)*10)/pi%2B0.5,+x%3D-1+to+1,+y%3D-1+to+1 +// s_mod = atan((s_mod-0.5)*10.0)/dealii::numbers::PI+0.5; + + return T1+(T0-T1)*(1.0-s_mod); + + //old: +// return T1+(T0-T1)*((1-s)*(1-s)); +// return T1+(T0-T1)*((1-s)); } @@ -224,6 +311,103 @@ namespace EquationData // name. namespace LinearSolvers { + template + class RightPrecond : public Subscriptor + { + public: + RightPrecond ( + const TrilinosWrappers::BlockSparseMatrix &S, + const TrilinosWrappers::BlockSparseMatrix &Spre, + const PreconditionerMp &Mppreconditioner, + const PreconditionerA &Apreconditioner) + : + stokes_matrix (&S), + stokes_preconditioner_matrix (&Spre), + mp_preconditioner (Mppreconditioner), + a_preconditioner (Apreconditioner) + {} + + void solve_S(TrilinosWrappers::MPI::Vector &dst, + const TrilinosWrappers::MPI::Vector &src) const + { + SolverControl cn(5000, 1e-5);//src.l2_norm()*1e-5); + + TrilinosWrappers::SolverBicgstab solver(cn); + + solver.solve(stokes_preconditioner_matrix->block(1,1), + dst, src, + mp_preconditioner); + +#ifdef CONF_LOG_SOLVER + TrilinosWrappers::MPI::Vector res(src); + double resid = + stokes_preconditioner_matrix->block(1,1).residual(res,dst,src); + + double srcl2 = src.l2_norm(); +#endif + dst*=-1.0; + +#ifdef CONF_LOG_SOLVER + int myid=Utilities::System::get_this_mpi_process(MPI_COMM_WORLD); + if (myid==0) + { + std::cout << " solve_S it=" << cn.last_step() + << " startres=" << stringify(srcl2) + << " res=" << stringify(resid) + << " reduction=" << stringify(resid / srcl2) + << std::endl; + + } +#endif + + } + + void solve_A(TrilinosWrappers::MPI::Vector &dst, + const TrilinosWrappers::MPI::Vector &src) const + { + SolverControl cn(5000, src.l2_norm()*1e-4); + TrilinosWrappers::SolverBicgstab solver(cn); + solver.solve(stokes_matrix->block(0,0), dst, src, a_preconditioner); + +#ifdef CONF_LOG_SOLVER + double srcl2 = src.l2_norm(); + TrilinosWrappers::MPI::Vector res(src); + double resid = stokes_matrix->block(0,0).residual(res,dst,src); + + int myid=Utilities::System::get_this_mpi_process(MPI_COMM_WORLD); + if (myid==0) + { + std::cout << " solve_A it=" << cn.last_step() + << " startres=" << stringify(srcl2) + << " res=" << stringify(resid) + << " reduction=" << stringify(resid / srcl2) + << std::endl; + + } +#endif + } + + void vmult (TrilinosWrappers::MPI::BlockVector &dst, + const TrilinosWrappers::MPI::BlockVector &src) const + { + TrilinosWrappers::MPI::Vector utmp(src.block(0)); + + solve_S(dst.block(1), src.block(1)); + + stokes_matrix->block(0,1).vmult(utmp, dst.block(1)); //B^T + utmp*=-1.0; + utmp.add(src.block(0)); + + solve_A(dst.block(0), utmp); + } + + private: + const SmartPointer stokes_matrix; + const SmartPointer stokes_preconditioner_matrix; + const PreconditionerMp &mp_preconditioner; + const PreconditionerA &a_preconditioner; + }; + template class BlockSchurPreconditioner : public Subscriptor { @@ -801,7 +985,7 @@ class BoussinesqFlowProblem { public: BoussinesqFlowProblem (); - void run (); + void run (unsigned int ref); private: void setup_dofs (); @@ -834,34 +1018,34 @@ class BoussinesqFlowProblem ConditionalOStream pcout; - Triangulation triangulation; + parallel::distributed::Triangulation triangulation; double global_Omega_diameter; const unsigned int stokes_degree; FESystem stokes_fe; - DoFHandler stokes_dof_handler; + DoFHandler stokes_dof_handler; ConstraintMatrix stokes_constraints; TrilinosWrappers::BlockSparseMatrix stokes_matrix; TrilinosWrappers::BlockSparseMatrix stokes_preconditioner_matrix; - TrilinosWrappers::BlockVector stokes_solution; - TrilinosWrappers::BlockVector old_stokes_solution; + TrilinosWrappers::MPI::BlockVector stokes_solution; + TrilinosWrappers::MPI::BlockVector old_stokes_solution; TrilinosWrappers::MPI::BlockVector stokes_rhs; const unsigned int temperature_degree; FE_Q temperature_fe; - DoFHandler temperature_dof_handler; + DoFHandler temperature_dof_handler; ConstraintMatrix temperature_constraints; TrilinosWrappers::SparseMatrix temperature_mass_matrix; TrilinosWrappers::SparseMatrix temperature_stiffness_matrix; TrilinosWrappers::SparseMatrix temperature_matrix; - TrilinosWrappers::Vector temperature_solution; - TrilinosWrappers::Vector old_temperature_solution; - TrilinosWrappers::Vector old_old_temperature_solution; + TrilinosWrappers::MPI::Vector temperature_solution; + TrilinosWrappers::MPI::Vector old_temperature_solution; + TrilinosWrappers::MPI::Vector old_old_temperature_solution; TrilinosWrappers::MPI::Vector temperature_rhs; @@ -977,7 +1161,11 @@ BoussinesqFlowProblem::BoussinesqFlowProblem () get_this_mpi_process(MPI_COMM_WORLD) == 0)), - triangulation (Triangulation::maximum_smoothing), + triangulation (MPI_COMM_WORLD, + typename Triangulation::MeshSmoothing + (Triangulation::smoothing_on_refinement | Triangulation::smoothing_on_coarsening + ) + ), stokes_degree (1), stokes_fe (FE_Q(stokes_degree+1), dim, @@ -1435,8 +1623,9 @@ void BoussinesqFlowProblem::project_temperature_field () cg.solve (temperature_mass_matrix, solution, rhs, preconditioner_mass); - old_temperature_solution = solution; - temperature_constraints.distribute (old_temperature_solution); + temperature_constraints.distribute (solution); +// old_temperature_solution = solution; + old_temperature_solution.reinit(solution, false, true); } @@ -1537,7 +1726,8 @@ void BoussinesqFlowProblem:: else coupling[c][d] = DoFTools::none; - DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp, + DoFTools::make_sparsity_pattern (static_cast&>(stokes_dof_handler), + coupling, sp, stokes_constraints, false, Utilities::System:: get_this_mpi_process(MPI_COMM_WORLD)); @@ -1568,7 +1758,8 @@ void BoussinesqFlowProblem:: else coupling[c][d] = DoFTools::none; - DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp, + DoFTools::make_sparsity_pattern (static_cast&>(stokes_dof_handler), + coupling, sp, stokes_constraints, false, Utilities::System:: get_this_mpi_process(MPI_COMM_WORLD)); @@ -1589,7 +1780,7 @@ void BoussinesqFlowProblem:: TrilinosWrappers::SparsityPattern sp (temperature_partitioner, MPI_COMM_WORLD); - DoFTools::make_sparsity_pattern (temperature_dof_handler, sp, + DoFTools::make_sparsity_pattern (static_cast&>(temperature_dof_handler), sp, temperature_constraints, false, Utilities::System:: get_this_mpi_process(MPI_COMM_WORLD)); @@ -1684,55 +1875,20 @@ template void BoussinesqFlowProblem::setup_dofs () { computing_timer.enter_section("Setup dof systems"); + TimeBlock t(pcout, "**dof_setup", false); + std::vector stokes_sub_blocks (dim+1,0); stokes_sub_blocks[dim] = 1; - - GridTools::partition_triangulation (Utilities::System:: - get_n_mpi_processes(MPI_COMM_WORLD), - triangulation); - { + TimeBlock t(pcout, "***dof_distribute"); + stokes_dof_handler.distribute_dofs (stokes_fe); - DoFRenumbering::subdomain_wise (stokes_dof_handler); DoFRenumbering::component_wise (stokes_dof_handler, stokes_sub_blocks); - stokes_constraints.clear (); - DoFTools::make_hanging_node_constraints (stokes_dof_handler, - stokes_constraints); - - std::vector velocity_mask (dim+1, true); - velocity_mask[dim] = false; - VectorTools::interpolate_boundary_values (stokes_dof_handler, - 0, - ZeroFunction(dim+1), - stokes_constraints, - velocity_mask); - - std::set no_normal_flux_boundaries; - no_normal_flux_boundaries.insert (1); - VectorTools::compute_no_normal_flux_constraints (stokes_dof_handler, 0, - no_normal_flux_boundaries, - stokes_constraints); - stokes_constraints.close (); - } - { temperature_dof_handler.distribute_dofs (temperature_fe); - DoFRenumbering::subdomain_wise (temperature_dof_handler); - - temperature_constraints.clear (); - VectorTools::interpolate_boundary_values (temperature_dof_handler, - 0, - EquationData::TemperatureInitialValues(), - temperature_constraints); - VectorTools::interpolate_boundary_values (temperature_dof_handler, - 1, - EquationData::TemperatureInitialValues(), - temperature_constraints); - DoFTools::make_hanging_node_constraints (temperature_dof_handler, - temperature_constraints); - temperature_constraints.close (); } + std::vector stokes_dofs_per_block (2); DoFTools::count_dofs_per_block (stokes_dof_handler, stokes_dofs_per_block, stokes_sub_blocks); @@ -1742,7 +1898,7 @@ void BoussinesqFlowProblem::setup_dofs () n_T = temperature_dof_handler.n_dofs(); pcout << "Number of active cells: " - << triangulation.n_active_cells() + << triangulation.n_global_active_cells() << " (on " << triangulation.n_levels() << " levels)" @@ -1753,20 +1909,80 @@ void BoussinesqFlowProblem::setup_dofs () << std::endl << std::endl; - std::vector stokes_partitioning; - IndexSet temperature_partitioning (n_T); + + + std::vector stokes_partitioning, stokes_relevant_partitioning; + IndexSet temperature_partitioning (n_T), temperature_relevant_partitioning (n_T); + IndexSet stokes_relevant_set; { + TimeBlock t(pcout, "***index_sets"); const unsigned int my_id = Utilities::System::get_this_mpi_process(MPI_COMM_WORLD); - IndexSet stokes_index_set = - DoFTools::dof_indices_with_subdomain_association(stokes_dof_handler, - my_id); + IndexSet stokes_index_set = stokes_dof_handler.locally_owned_dofs(); stokes_partitioning.push_back(stokes_index_set.get_view(0,n_u)); stokes_partitioning.push_back(stokes_index_set.get_view(n_u,n_u+n_p)); - temperature_partitioning = - DoFTools::dof_indices_with_subdomain_association(temperature_dof_handler, - my_id); + DoFTools::extract_locally_relevant_dofs (stokes_dof_handler, + stokes_relevant_set); + stokes_relevant_partitioning.push_back(stokes_relevant_set.get_view(0,n_u)); + stokes_relevant_partitioning.push_back(stokes_relevant_set.get_view(n_u,n_u+n_p)); + + temperature_partitioning = temperature_dof_handler.locally_owned_dofs(); + DoFTools::extract_locally_relevant_dofs (temperature_dof_handler, + temperature_relevant_partitioning); + } + + { + + stokes_constraints.clear (); +// IndexSet stokes_la; +// DoFTools::extract_locally_active_dofs (stokes_dof_handler, +// stokes_la); + stokes_constraints.reinit(stokes_relevant_set); + + { + TimeBlock t(pcout, "***make_hanging_nodes_vel"); + + DoFTools::make_hanging_node_constraints (static_cast&>(stokes_dof_handler), + stokes_constraints); + } + + TimeBlock t(pcout, "***boundary_values_vel"); + + std::vector velocity_mask (dim+1, true); + velocity_mask[dim] = false; + VectorTools::interpolate_boundary_values (static_cast&>(stokes_dof_handler), + 0, + ZeroFunction(dim+1), + stokes_constraints, + velocity_mask); + + std::set no_normal_flux_boundaries; + no_normal_flux_boundaries.insert (1); + VectorTools::compute_no_normal_flux_constraints (static_cast&>(stokes_dof_handler), 0, + no_normal_flux_boundaries, + stokes_constraints); + stokes_constraints.close (); + } + { + TimeBlock t(pcout, "***hanging_nodes_and_bv_temperature"); + temperature_constraints.clear (); +// IndexSet temp_locally_active; +// DoFTools::extract_locally_active_dofs (temperature_dof_handler, +// temp_locally_active); + temperature_constraints.reinit(temperature_relevant_partitioning);//temp_locally_active); + + VectorTools::interpolate_boundary_values (static_cast&>(temperature_dof_handler), + 0, + EquationData::TemperatureInitialValues(), + temperature_constraints); + VectorTools::interpolate_boundary_values (static_cast&>(temperature_dof_handler), + 1, + EquationData::TemperatureInitialValues(), + temperature_constraints); + DoFTools::make_hanging_node_constraints (static_cast&>(temperature_dof_handler), + temperature_constraints); + temperature_constraints.close (); } if (Utilities::System::job_supports_mpi() == false) @@ -1785,17 +2001,26 @@ void BoussinesqFlowProblem::setup_dofs () } else { + TimeBlock t(pcout, "***setup_stokes_matrix"); setup_stokes_matrix (stokes_partitioning); + t.close(); + + TimeBlock t2(pcout, "***setup_stokes_preconditioner"); setup_stokes_preconditioner (stokes_partitioning); + t2.close(); + + TimeBlock t3(pcout, "***setup_temperature_matrix"); setup_temperature_matrices (temperature_partitioning); } + TimeBlock t2(pcout, "***init_vectors"); + stokes_rhs.reinit (stokes_partitioning, MPI_COMM_WORLD); - stokes_solution.reinit (stokes_rhs); + stokes_solution.reinit (stokes_relevant_partitioning, MPI_COMM_WORLD); old_stokes_solution.reinit (stokes_solution); temperature_rhs.reinit (temperature_partitioning, MPI_COMM_WORLD); - temperature_solution.reinit (temperature_rhs); + temperature_solution.reinit (temperature_relevant_partitioning, MPI_COMM_WORLD); old_temperature_solution.reinit (temperature_solution); old_old_temperature_solution.reinit (temperature_solution); @@ -1890,7 +2115,7 @@ local_assemble_stokes_preconditioner (const typename DoFHandler::active_cel (1./EquationData::eta) * EquationData::pressure_scaling * EquationData::pressure_scaling * - scratch.phi_p[i] * scratch.phi_p[j]) + (scratch.phi_p[i] * scratch.phi_p[j])) * scratch.stokes_fe_values.JxW(q); } } @@ -2016,11 +2241,13 @@ template void BoussinesqFlowProblem::build_stokes_preconditioner () { + TimeBlock t(pcout, "*build_stokes_preconditioner"); + if (rebuild_stokes_preconditioner == false) return; - computing_timer.enter_section (" Build Stokes preconditioner"); - pcout << " Rebuilding Stokes preconditioner..." << std::flush; +// computing_timer.enter_section (" Build Stokes preconditioner"); +// pcout << " Rebuilding Stokes preconditioner..." << std::flush; assemble_stokes_preconditioner (); @@ -2040,7 +2267,7 @@ BoussinesqFlowProblem::build_stokes_preconditioner () Amg_data.elliptic = true; Amg_data.higher_order_elements = true; Amg_data.smoother_sweeps = 2; - Amg_data.aggregation_threshold = 0.02; +// Amg_data.aggregation_threshold = 0.02; Mp_preconditioner->initialize (stokes_preconditioner_matrix.block(1,1)); Amg_preconditioner->initialize (stokes_preconditioner_matrix.block(0,0), @@ -2048,8 +2275,8 @@ BoussinesqFlowProblem::build_stokes_preconditioner () rebuild_stokes_preconditioner = false; - pcout << std::endl; - computing_timer.exit_section(); +// pcout << std::endl; +// computing_timer.exit_section(); } // @sect5{Stokes system assembly} @@ -2166,9 +2393,9 @@ copy_local_to_global_stokes_system (const Assembly::CopyData::StokesSystem template void BoussinesqFlowProblem::assemble_stokes_system () { - pcout << " Assembling..." << std::flush; + TimeBlock t(pcout, "*assemble_stokes"); - computing_timer.enter_section (" Assemble Stokes system"); +// computing_timer.enter_section (" Assemble Stokes system"); if (rebuild_stokes_matrix == true) stokes_matrix=0; @@ -2218,8 +2445,8 @@ void BoussinesqFlowProblem::assemble_stokes_system () rebuild_stokes_matrix = false; - pcout << std::endl; - computing_timer.exit_section(); +// pcout << std::endl; +// computing_timer.exit_section(); } @@ -2295,10 +2522,11 @@ copy_local_to_global_temperature_matrix (const Assembly::CopyData::TemperatureMa template void BoussinesqFlowProblem::assemble_temperature_matrix () { + TimeBlock t(pcout, "*assemble_temp_matrix"); if (rebuild_temperature_matrices == false) return; - computing_timer.enter_section (" Assemble temperature matrices"); +// computing_timer.enter_section (" Assemble temperature matrices"); temperature_mass_matrix = 0; temperature_stiffness_matrix = 0; @@ -2336,7 +2564,7 @@ void BoussinesqFlowProblem::assemble_temperature_matrix () rebuild_temperature_matrices = false; rebuild_temperature_preconditioner = true; - computing_timer.exit_section(); +// computing_timer.exit_section(); } @@ -2658,17 +2886,26 @@ void BoussinesqFlowProblem::assemble_temperature_system (const double maxim template void BoussinesqFlowProblem::solve () { - computing_timer.enter_section (" Solve Stokes system"); - pcout << " Solving..." << std::endl; +// computing_timer.enter_section (" Solve Stokes system"); { + TimeBlock t(pcout, "*solve_stokes", false); + const LinearSolvers::BlockSchurPreconditioner preconditioner (stokes_matrix, *Mp_preconditioner, *Amg_preconditioner); + const LinearSolvers::RightPrecond + preconditioner_right (stokes_matrix, stokes_preconditioner_matrix, + *Mp_preconditioner, *Amg_preconditioner); + TrilinosWrappers::MPI::BlockVector distributed_stokes_solution (stokes_rhs); - distributed_stokes_solution = stokes_solution; +// distributed_stokes_solution = stokes_solution; + distributed_stokes_solution.block(0).reinit(stokes_solution.block(0),false,true); + distributed_stokes_solution.block(1).reinit(stokes_solution.block(1),false,true); + const unsigned int start = (distributed_stokes_solution.block(0).size() + @@ -2679,39 +2916,94 @@ void BoussinesqFlowProblem::solve () if (stokes_constraints.is_constrained (i)) distributed_stokes_solution(i) = 0; - SolverControl solver_control (stokes_matrix.m(), 1e-22*stokes_rhs.l2_norm()); - SolverBicgstab - bicgstab (solver_control, false); +#ifdef CONF_COMPARE_SOLVER + { + TrilinosWrappers::MPI::BlockVector x(distributed_stokes_solution); + { + TimeBlock t(pcout, "*TMP: solve bicg", false); + + PrimitiveVectorMemory< TrilinosWrappers::MPI::BlockVector > mem; + SolverControl solver_control (stokes_matrix.m(), 1e-24*stokes_rhs.l2_norm()); + SolverBicgstab + solver (solver_control, mem, false); + solver.solve(stokes_matrix, x, stokes_rhs, + preconditioner); + + pcout << " xx BICG: " + << solver_control.last_step() + << " iterations," + << " reduced res by " << stringify(solver_control.last_value()/solver_control.initial_value()) + << std::endl; + } + { + TrilinosWrappers::MPI::BlockVector + res (stokes_rhs); + double residual = stokes_matrix.residual(res, x, stokes_rhs); - bicgstab.solve(stokes_matrix, distributed_stokes_solution, stokes_rhs, - preconditioner); + pcout << " x BICG: real reduced res by " << stringify(residual + /stokes_rhs.l2_norm()) + << std::endl; + } + } +#endif + + SolverControl solver_control (stokes_matrix.m(), 1e-7*stokes_rhs.l2_norm()); + { +#ifdef CONF_COMPARE_SOLVER + TimeBlock t(pcout, "*TMP: solve fgmres", false); +#endif + PrimitiveVectorMemory< TrilinosWrappers::MPI::BlockVector > mem; + SolverFGMRES + solver(solver_control, mem, + SolverFGMRES::AdditionalData(50, true)); + solver.solve(stokes_matrix, distributed_stokes_solution, stokes_rhs, + preconditioner_right); + + pcout << " xx FGMRES: " + << solver_control.last_step() + << " iterations," + << " reduced res by " << stringify(solver_control.last_value()/solver_control.initial_value()) + << std::endl; + } + { + TrilinosWrappers::MPI::BlockVector + res (stokes_rhs); + double residual = stokes_matrix.residual(res, distributed_stokes_solution, stokes_rhs); + + pcout << " x FGMRES: real reduced res by " << stringify(residual + /stokes_rhs.l2_norm()) + << std::endl; + } + + stokes_constraints.distribute (distributed_stokes_solution); + //stokes_solution = distributed_stokes_solution; + stokes_solution.block(0).reinit(distributed_stokes_solution.block(0), false, true); + stokes_solution.block(1).reinit(distributed_stokes_solution.block(1), false, true); - stokes_solution = distributed_stokes_solution; pcout << " " << solver_control.last_step() - << " BiCGStab iterations for Stokes subsystem." + << " iterations for Stokes subsystem." + << " reduced res by " << stringify(solver_control.last_value()/solver_control.initial_value()) << std::endl; - stokes_constraints.distribute (stokes_solution); + } - computing_timer.exit_section(); +// computing_timer.exit_section(); - computing_timer.enter_section (" Assemble temperature rhs"); +// computing_timer.enter_section (" Assemble temperature rhs"); old_time_step = time_step; const double maximal_velocity = get_maximal_velocity(); - if (maximal_velocity > 1e-10) - time_step = 1./(1.6*dim*std::sqrt(1.*dim)) / - temperature_degree * - GridTools::minimal_cell_diameter(triangulation) / - maximal_velocity; - else - time_step = 1./(1.6*dim*std::sqrt(1.*dim)) / - temperature_degree * - GridTools::minimal_cell_diameter(triangulation) / - 1e-10; + double local_time_step = 1./(1.6*dim*std::sqrt(1.*dim)) / + temperature_degree * + GridTools::minimal_cell_diameter(triangulation) / + std::max(1e-10,maximal_velocity); + + // calculate the minimum allowed time step + // size + MPI_Allreduce (&local_time_step, &time_step, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); pcout << " Maximal velocity: " << maximal_velocity * EquationData::year_in_seconds * 100 @@ -2723,51 +3015,69 @@ void BoussinesqFlowProblem::solve () << std::endl; temperature_solution = old_temperature_solution; + { + TimeBlock t(pcout, "*assemble_temp"); - assemble_temperature_system (maximal_velocity); + assemble_temperature_system (maximal_velocity); + } - computing_timer.exit_section (); - computing_timer.enter_section (" Solve temperature system"); +// computing_timer.exit_section (); + +// computing_timer.enter_section (" Solve temperature system"); { + TimeBlock t(pcout, "*solve_temp", false); SolverControl solver_control (temperature_matrix.m(), 1e-12*temperature_rhs.l2_norm()); SolverCG cg (solver_control); TrilinosWrappers::MPI::Vector distributed_temperature_solution (temperature_rhs); - distributed_temperature_solution = temperature_solution; +// distributed_temperature_solution = temperature_solution; + distributed_temperature_solution.reinit(temperature_solution, false, true); cg.solve (temperature_matrix, distributed_temperature_solution, temperature_rhs, *T_preconditioner); - temperature_solution = distributed_temperature_solution; - temperature_constraints.distribute (temperature_solution); + temperature_constraints.distribute (distributed_temperature_solution); +// temperature_solution = distributed_temperature_solution; + temperature_solution.reinit(distributed_temperature_solution, false, true); pcout << " " << solver_control.last_step() << " CG iterations for temperature" << std::endl; - computing_timer.exit_section(); +// computing_timer.exit_section(); - double min_temperature = temperature_solution(0), - max_temperature = temperature_solution(0); - for (unsigned int i=1; i temperature (2), global_temperature (2); + temperature[0] = distributed_temperature_solution.trilinos_vector()[0][0], + temperature[1] = temperature[0]; + for (unsigned int i=1; i (min_temperature, - temperature_solution(i)); - max_temperature = std::max (max_temperature, - temperature_solution(i)); + temperature[0] = std::min (temperature[0], + distributed_temperature_solution.trilinos_vector()[0][i]); + temperature[1] = std::max (temperature[1], + distributed_temperature_solution.trilinos_vector()[0][i]); } +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + temperature[0] *= -1.0; + MPI_Allreduce (&temperature[0], &global_temperature[0], + 2, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + global_temperature[0] *= -1.0; +#else + global_temperature = local_temperature; +#endif pcout << " Temperature range: " - << min_temperature << ' ' << max_temperature + << global_temperature[0] << ' ' << global_temperature[1] << std::endl; } } + // @sect4{BoussinesqFlowProblem::output_results} // This function does mostly what the @@ -2839,49 +3149,61 @@ void BoussinesqFlowProblem::solve () template void BoussinesqFlowProblem::output_results () { - if (timestep_number % 25 != 0) - return; + pcout << "outputting " << out_index << std::endl; computing_timer.enter_section ("Postprocessing"); + Vector estimated_error_per_cell (triangulation.n_active_cells()); + + KellyErrorEstimator::estimate (static_cast&>(temperature_dof_handler), + QGauss(temperature_degree+1), + typename FunctionMap::type(), + temperature_solution, + estimated_error_per_cell); + + const FESystem joint_fe (stokes_fe, 1, + temperature_fe, 1, + FE_DGQ(0), 1, + FE_DGQ(0), 1, + temperature_fe, 1); + + DoFHandler joint_dof_handler (triangulation); + joint_dof_handler.distribute_dofs (joint_fe); + Assert (joint_dof_handler.n_global_dofs() == + stokes_dof_handler.n_global_dofs() + + temperature_dof_handler.n_global_dofs() + + 2*triangulation.n_global_active_cells() + + temperature_dof_handler.n_global_dofs(), + ExcInternalError()); + + TrilinosWrappers::MPI::Vector joint_solution; + IndexSet locally_relevant_dofs(joint_dof_handler.n_dofs()); + DoFTools::extract_locally_relevant_dofs (joint_dof_handler, locally_relevant_dofs); + joint_solution.reinit (locally_relevant_dofs, MPI_COMM_WORLD); - if (Utilities::System::get_this_mpi_process(MPI_COMM_WORLD) == 0) - { - - const FESystem joint_fe (stokes_fe, 1, - temperature_fe, 1, - FE_DGQ(0), 1); - DoFHandler joint_dof_handler (triangulation); - joint_dof_handler.distribute_dofs (joint_fe); - Assert (joint_dof_handler.n_dofs() == - stokes_dof_handler.n_dofs() + - temperature_dof_handler.n_dofs() + - triangulation.n_active_cells(), - ExcInternalError()); - - Vector joint_solution (joint_dof_handler.n_dofs()); - - { - double minimal_pressure = stokes_solution.block(1)(0); - for (unsigned int i=0; i (stokes_solution.block(1)(i), - minimal_pressure); - - std::vector local_joint_dof_indices (joint_fe.dofs_per_cell); - std::vector local_stokes_dof_indices (stokes_fe.dofs_per_cell); - std::vector local_temperature_dof_indices (temperature_fe.dofs_per_cell); - - typename DoFHandler::active_cell_iterator - joint_cell = joint_dof_handler.begin_active(), - joint_endc = joint_dof_handler.end(), - stokes_cell = stokes_dof_handler.begin_active(), - temperature_cell = temperature_dof_handler.begin_active(); - for (; joint_cell!=joint_endc; ++joint_cell, ++stokes_cell, ++temperature_cell) - { - joint_cell->get_dof_indices (local_joint_dof_indices); - stokes_cell->get_dof_indices (local_stokes_dof_indices); - temperature_cell->get_dof_indices (local_temperature_dof_indices); + { + //double minimal_pressure = stokes_solution.block(1)(0); + //for (unsigned int i=0; i (stokes_solution.block(1)(i), + // minimal_pressure); + + std::vector local_joint_dof_indices (joint_fe.dofs_per_cell); + std::vector local_stokes_dof_indices (stokes_fe.dofs_per_cell); + std::vector local_temperature_dof_indices (temperature_fe.dofs_per_cell); + + typename DoFHandler::active_cell_iterator + joint_cell = joint_dof_handler.begin_active(), + joint_endc = joint_dof_handler.end(), + stokes_cell = stokes_dof_handler.begin_active(), + temperature_cell = temperature_dof_handler.begin_active(); + for (unsigned int cell_index=0; joint_cell!=joint_endc; ++joint_cell, ++stokes_cell, ++temperature_cell, ++cell_index) + if (!joint_cell->is_artificial() && !joint_cell->is_ghost()) + { + joint_cell->get_dof_indices (local_joint_dof_indices); + stokes_cell->get_dof_indices (local_stokes_dof_indices); + temperature_cell->get_dof_indices (local_temperature_dof_indices); - for (unsigned int i=0; i::output_results () joint_solution(local_joint_dof_indices[i]) = ((stokes_solution(local_stokes_dof_indices [joint_fe.system_to_base_index(i).second]) - - - minimal_pressure) + ) * EquationData::pressure_scaling); } } - else if (joint_fe.system_to_base_index(i).first.first == 1) - { - Assert (joint_fe.system_to_base_index(i).second - < - local_temperature_dof_indices.size(), - ExcInternalError()); - joint_solution(local_joint_dof_indices[i]) - = temperature_solution(local_temperature_dof_indices - [joint_fe.system_to_base_index(i).second]); - } + else if (joint_fe.system_to_base_index(i).first.first == 1) + { + Assert (joint_fe.system_to_base_index(i).second + < + local_temperature_dof_indices.size(), + ExcInternalError()); + joint_solution(local_joint_dof_indices[i]) + = temperature_solution(local_temperature_dof_indices + [joint_fe.system_to_base_index(i).second]); + } + else if (joint_fe.system_to_base_index(i).first.first == 2) + { + Assert (joint_fe.system_to_base_index(i).second + == 0, + ExcInternalError()); + joint_solution(local_joint_dof_indices[i]) + = joint_cell->subdomain_id(); + } + else if (joint_fe.system_to_base_index(i).first.first == 3) + { + Assert (joint_fe.system_to_base_index(i).first.first == 3, + ExcInternalError()); + Assert (joint_fe.system_to_base_index(i).second + == 0, + ExcInternalError()); + joint_solution(local_joint_dof_indices[i]) + = estimated_error_per_cell (cell_index); + } else { - Assert (joint_fe.system_to_base_index(i).first.first == 2, - ExcInternalError()); - Assert (joint_fe.system_to_base_index(i).second - == 0, - ExcInternalError()); - joint_solution(local_joint_dof_indices[i]) - = joint_cell->subdomain_id(); - } - } - } + Assert (joint_fe.system_to_base_index(i).first.first == 4, + ExcInternalError()); - std::vector joint_solution_names (dim, "velocity"); - joint_solution_names.push_back ("p"); - joint_solution_names.push_back ("T"); - joint_solution_names.push_back ("partition"); + joint_solution(local_joint_dof_indices[i]) + = local_temperature_dof_indices + [joint_fe.system_to_base_index(i).second]; - DataOut data_out; - - data_out.attach_dof_handler (joint_dof_handler); - - std::vector - data_component_interpretation - (dim+3, DataComponentInterpretation::component_is_scalar); - for (unsigned int i=0; i::type_dof_data, - data_component_interpretation); - data_out.build_patches (std::min(stokes_degree, temperature_degree)); + } + } + } - std::ostringstream filename; - filename << "solution-" << Utilities::int_to_string(timestep_number, 5) - << ".vtk"; + std::vector joint_solution_names (dim, "velocity"); + joint_solution_names.push_back ("p"); + joint_solution_names.push_back ("T"); + joint_solution_names.push_back ("partition"); + joint_solution_names.push_back ("error"); + joint_solution_names.push_back ("TDofIndex"); + + DataOut data_out; + + data_out.attach_dof_handler (joint_dof_handler); + + std::vector + data_component_interpretation + (dim+5, DataComponentInterpretation::component_is_scalar); + for (unsigned int i=0; i::type_dof_data, + data_component_interpretation); + data_out.build_patches (1+std::min(stokes_degree, temperature_degree)); + + const std::string filename = ("out/solution-" + + Utilities::int_to_string (out_index, 5) + + "." + + Utilities::int_to_string + (triangulation.locally_owned_subdomain(), 4) + + ".vtu"); + std::ofstream output (filename.c_str()); + data_out.write_vtu (output); - std::ofstream output (filename.str().c_str()); - data_out.write_vtk (output); + if (Utilities::System::get_this_mpi_process(MPI_COMM_WORLD) == 0) + { + std::vector filenames; + for (unsigned int i=0; i::output_results () template void BoussinesqFlowProblem::refine_mesh (const unsigned int max_grid_level) { + TimeBlock t(pcout, "*refine_mesh", false); + computing_timer.enter_section ("Refine mesh structure, part 1"); - Vector local_estimated_error_per_cell (triangulation.n_active_cells()); + Vector estimated_error_per_cell (triangulation.n_active_cells()); + + { + TimeBlock t(pcout, "**kelly"); - KellyErrorEstimator::estimate (temperature_dof_handler, + KellyErrorEstimator::estimate (static_cast&>(temperature_dof_handler), QGauss(temperature_degree+1), typename FunctionMap::type(), temperature_solution, - local_estimated_error_per_cell, + estimated_error_per_cell, std::vector(), 0, 0, - Utilities::System::get_this_mpi_process(MPI_COMM_WORLD)); + triangulation.locally_owned_subdomain()); + + } + + +#ifdef CONF_BENCHMARK + /* pure refinement to double the number of + cells in benchmark mode if this is + prerefinement. Keep the number of cells + constant later on. */ + if (timestep_number==0) + { + pcout << "refine_mesh: prerefine, factor 2." << std::endl; + parallel::distributed::GridRefinement:: + refine_and_coarsen_fixed_number (triangulation, + estimated_error_per_cell, + 0.3333, 0.0); + } + else + { + pcout << "refine_mesh: keeping mesh constant." << std::endl; + TimeBlock t(pcout, "**refine&coarsen_fixed_nb"); + + parallel::distributed::GridRefinement:: + refine_and_coarsen_fixed_number (triangulation, + estimated_error_per_cell, + 0.3333, 0.0);//0.1, 0.4 ? + } - Vector x_local_estimated_error_per_cell (triangulation.n_active_cells()); - x_local_estimated_error_per_cell = local_estimated_error_per_cell; - Vector estimated_error_per_cell (triangulation.n_active_cells()); -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - MPI_Allreduce (&x_local_estimated_error_per_cell(0), - &estimated_error_per_cell(0), - triangulation.n_active_cells(), MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); #else - estimated_error_per_cell = x_local_estimated_error_per_cell; -#endif + parallel::distributed::GridRefinement:: + refine_and_coarsen_fixed_fraction (triangulation, + estimated_error_per_cell, + 0.6, 0.2); - GridRefinement::refine_and_coarsen_fixed_fraction (triangulation, - estimated_error_per_cell, - 0.6, 0.2); + // limit maximum refinement level if (triangulation.n_levels() > max_grid_level) for (typename Triangulation::active_cell_iterator cell = triangulation.begin_active(max_grid_level); cell != triangulation.end(); ++cell) cell->clear_refine_flag (); +#endif - std::vector x_temperature (2); - x_temperature[0] = temperature_solution; - x_temperature[1] = old_temperature_solution; - TrilinosWrappers::BlockVector x_stokes = stokes_solution; + TimeBlock t_a(pcout, "**prepare_solution_transfer", false); + std::vector x_temperature (2); + x_temperature[0] = &temperature_solution; + x_temperature[1] = &old_temperature_solution; + TrilinosWrappers::MPI::BlockVector x_stokes = stokes_solution; - SolutionTransfer + parallel::distributed::SolutionTransfer temperature_trans(temperature_dof_handler); - SolutionTransfer + parallel::distributed::SolutionTransfer stokes_trans(stokes_dof_handler); triangulation.prepare_coarsening_and_refinement(); temperature_trans.prepare_for_coarsening_and_refinement(x_temperature); stokes_trans.prepare_for_coarsening_and_refinement(x_stokes); + t_a.close(); + TimeBlock t_c(pcout, "**execute_c&r", false); + triangulation.execute_coarsening_and_refinement (); computing_timer.exit_section(); + t_c.close(); + setup_dofs (); + TimeBlock t_b(pcout, "**do_solution_transfer"); computing_timer.enter_section ("Refine mesh structure, part 2"); - std::vector tmp (2); - tmp[0].reinit (temperature_solution); - tmp[1].reinit (temperature_solution); - temperature_trans.interpolate(x_temperature, tmp); + TrilinosWrappers::MPI::Vector + distributed_temp1 (temperature_rhs); + TrilinosWrappers::MPI::Vector + distributed_temp2 (temperature_rhs); + + std::vector tmp (2); + tmp[0] = &(distributed_temp1); + tmp[1] = &(distributed_temp2); + temperature_trans.interpolate(tmp); + +// temperature_solution = distributed_temp1; + temperature_solution.reinit(distributed_temp1, false, true); +// old_temperature_solution = distributed_temp2; + old_temperature_solution.reinit(distributed_temp2, false, true); + + TrilinosWrappers::MPI::BlockVector + distributed_stokes (stokes_rhs); - temperature_solution = tmp[0]; - old_temperature_solution = tmp[1]; - temperature_constraints.distribute(temperature_solution); - temperature_constraints.distribute(old_temperature_solution); + stokes_trans.interpolate (distributed_stokes); +// stokes_solution = distributed_stokes; + stokes_solution.block(0).reinit(distributed_stokes.block(0), false, true); + stokes_solution.block(1).reinit(distributed_stokes.block(1), false, true); - TrilinosWrappers::BlockVector x_stokes_new = stokes_solution; - stokes_trans.interpolate (x_stokes, x_stokes_new); - stokes_solution = x_stokes_new; rebuild_stokes_matrix = true; rebuild_stokes_preconditioner = true; @@ -3159,10 +3564,17 @@ void BoussinesqFlowProblem::refine_mesh (const unsigned int max_grid_level) // VectorTools::project, the // rest is as before. template -void BoussinesqFlowProblem::run () +void BoussinesqFlowProblem::run (unsigned int ref) { - const unsigned int initial_refinement = (dim == 2 ? 5 : 2); - const unsigned int n_pre_refinement_steps = (dim == 2 ? 2 : 2); + pcout << "this is step-32. ref=" << ref << std::endl; + #ifdef CONF_BENCHMARK + // create approximately 12*4^(ref/2) cells: + const unsigned int initial_refinement = (ref-1)/2; + const unsigned int n_pre_refinement_steps = 2+(ref-1)%2; +#else + const unsigned int initial_refinement = ref;//(dim == 2 ? 5 : 2); + const unsigned int n_pre_refinement_steps = 2;//(dim == 2 ? 4 : 2); +#endif GridGenerator::hyper_shell (triangulation, Point(), @@ -3170,11 +3582,12 @@ void BoussinesqFlowProblem::run () EquationData::R1, 12, true); - static HyperShellBoundary boundary; triangulation.set_boundary (0, boundary); triangulation.set_boundary (1, boundary); + //GridGenerator::hyper_cube (triangulation, EquationData::R0, EquationData::R1); + global_Omega_diameter = GridTools::diameter (triangulation); triangulation.refine_global (initial_refinement); @@ -3207,6 +3620,18 @@ void BoussinesqFlowProblem::run () pcout << std::endl; +#ifdef CONF_BENCHMARK + static int bench_step=0; + if (time>=1e5*EquationData::year_in_seconds) + { + ++bench_step; + if (bench_step>1) + break; + pcout << "***BEGIN BENCHSTEP" << std::endl; + refine_mesh (initial_refinement + n_pre_refinement_steps); + } + else +#endif if ((timestep_number == 0) && (pre_refinement_step < n_pre_refinement_steps)) { @@ -3214,11 +3639,19 @@ void BoussinesqFlowProblem::run () ++pre_refinement_step; goto start_time_iteration; } +#ifndef CONF_BENCHMARK else if ((timestep_number > 0) && (timestep_number % 10 == 0)) refine_mesh (initial_refinement + n_pre_refinement_steps); +#endif - output_results (); +#ifdef CONF_BENCHMARK + if (false)//bench_step) +#else + if (timestep_number % 10 == 0 && + Utilities::System::get_n_mpi_processes(MPI_COMM_WORLD) <= 10) +#endif + output_results (); time += time_step; ++timestep_number; @@ -3244,7 +3677,15 @@ int main (int argc, char *argv[]) Utilities::System::MPI_InitFinalize mpi_initialization(argc, argv); BoussinesqFlowProblem<2> flow_problem; - flow_problem.run (); + + unsigned int ref=5; + if (argc>=2) + { + ref = (unsigned int)Utilities::string_to_int(argv[1]); + } + + flow_problem.run (ref); + print_memory_stats(); } catch (std::exception &exc) { diff --git a/deal.II/examples/step-32/timeblock.h b/deal.II/examples/step-32/timeblock.h new file mode 100644 index 0000000000..0aaa5ffab3 --- /dev/null +++ b/deal.II/examples/step-32/timeblock.h @@ -0,0 +1,64 @@ +/** + * copyright: + * Research Group on Numerical Methods for PDEs + * University of Göttingen + * + * lube@math.uni-goettingen.de + */ + +#ifndef TIMEBLOCK_H +#define TIMEBLOCK_H + +template +class TimeBlock +{ + public: + TimeBlock(STREAM & stream_, const char* blockname, bool singleline=true) + : m_blockname(blockname), + m_singleline(singleline), + m_running(true), + m_timer(MPI_COMM_WORLD, true), + stream(stream_) + { + MPI_Barrier(MPI_COMM_WORLD); + stream << blockname << " ... "; + if (singleline) + stream << std::flush; + else + stream << std::endl; + m_timer.start(); + } + + ~TimeBlock() + { + close(); + } + + void close() + { + if (!m_running) + return; + m_running = false; + + m_timer.stop(); + + if (!m_singleline) + stream << m_blockname << " took "; + + stream << std::fixed << std::setprecision(4); + + m_timer.print_data(stream); + + MPI_Barrier(MPI_COMM_WORLD); + } + + + private: + const char* m_blockname; + bool m_singleline; + bool m_running; + dealii::Timer m_timer; + STREAM & stream; +}; + +#endif diff --git a/deal.II/examples/step-40/Makefile b/deal.II/examples/step-40/Makefile new file mode 100644 index 0000000000..083bb751a1 --- /dev/null +++ b/deal.II/examples/step-40/Makefile @@ -0,0 +1,191 @@ +# $Id$ + + +# For the small projects Makefile, you basically need to fill in only +# four fields. +# +# The first is the name of the application. It is assumed that the +# application name is the same as the base file name of the single C++ +# file from which the application is generated. +target = $(basename $(shell echo step-*.cc)) + +# The second field determines whether you want to run your program in +# debug or optimized mode. The latter is significantly faster, but no +# run-time checking of parameters and internal states is performed, so +# you should set this value to `on' while you develop your program, +# and to `off' when running production computations. +debug-mode = on + + +# As third field, we need to give the path to the top-level deal.II +# directory. You need to adjust this to your needs. Since this path is +# probably the most often needed one in the Makefile internals, it is +# designated by a single-character variable, since that can be +# reference using $D only, i.e. without the parentheses that are +# required for most other parameters, as e.g. in $(target). +D = ../../ + + +# The last field specifies the names of data and other files that +# shall be deleted when calling `make clean'. Object and backup files, +# executables and the like are removed anyway. Here, we give a list of +# files in the various output formats that deal.II supports. +clean-up-files = *gmv *gnuplot *gpl *eps *pov *vtk *d2 + + + + +# +# +# Usually, you will not need to change anything beyond this point. +# +# +# The next statement tell the `make' program where to find the +# deal.II top level directory and to include the file with the global +# settings +include $D/common/Make.global_options + +################################################################ +# This example program will only work if PETSc is installed. If this +# is not the case, then simply redefine the main targets to do nothing +# +# The same holds true if we use threads: PETSc and threads doesn't +# seem to work well together... +ifneq ($(USE_CONTRIB_PETSC)$(enable-threads),yesno) + ifneq ($(USE_CONTRIB_PETSC),yes) +default run clean: + @echo + @echo "===========================================================" + @echo "= This program cannot be compiled without PETSc. Make =" + @echo "= sure you have PETSc installed and detected during =" + @echo "= configuration of deal.II =" + @echo "===========================================================" + @echo + else +default run clean: + @echo + @echo "===========================================================" + @echo "= PETSc can not be used when running programs on =" + @echo "= multiple threads. Make sure you have specified the =" + @echo "= --disable-threads flag upon configuration of deal.II =" + @echo "===========================================================" + @echo + endif +else +# +################################################################ + + + + +# Since the whole project consists of only one file, we need not +# consider difficult dependencies. We only have to declare the +# libraries which we want to link to the object file, and there need +# to be two sets of libraries: one for the debug mode version of the +# application and one for the optimized mode. Here we have selected +# the versions for 2d. Note that the order in which the libraries are +# given here is important and that your applications won't link +# properly if they are given in another order. +# +# You may need to augment the lists of libraries when compiling your +# program for other dimensions, or when using third party libraries +libs.g = $(lib-deal2-2d.g) \ + $(lib-deal2-3d.g) \ + $(lib-lac.g) \ + $(lib-base.g) +libs.o = $(lib-deal2-2d.o) \ + $(lib-deal2-3d.o) \ + $(lib-lac.o) \ + $(lib-base.o) + + +# We now use the variable defined above which switch between debug and +# optimized mode to select the set of libraries to link with. Included +# in the list of libraries is the name of the object file which we +# will produce from the single C++ file. Note that by default we use +# the extension .g.o for object files compiled in debug mode and .o for +# object files in optimized mode (or whatever the local default on your +# system is instead of .o). +ifeq ($(debug-mode),on) + libraries = $(target).g.$(OBJEXT) $(libs.g) +else + libraries = $(target).$(OBJEXT) $(libs.o) +endif + + +# Now comes the first production rule: how to link the single object +# file produced from the single C++ file into the executable. Since +# this is the first rule in the Makefile, it is the one `make' selects +# if you call it without arguments. +$(target) : $(libraries) + @echo ============================ Linking $@ + @$(CXX) -o $@$(EXEEXT) $^ $(LIBS) $(LDFLAGS) + + +# To make running the application somewhat independent of the actual +# program name, we usually declare a rule `run' which simply runs the +# program. You can then run it by typing `make run'. This is also +# useful if you want to call the executable with arguments which do +# not change frequently. You may then want to add them to the +# following rule: +run: $(target) + @echo ============================ Running $< + @./$(target)$(EXEEXT) + + + +# As a last rule to the `make' program, we define what to do when +# cleaning up a directory. This usually involves deleting object files +# and other automatically created files such as the executable itself, +# backup files, and data files. Since the latter are not usually quite +# diverse, you needed to declare them at the top of this file. +clean: + -rm -f *.$(OBJEXT) *~ Makefile.dep $(target)$(EXEEXT) $(clean-up-files) + + +# Since we have not yet stated how to make an object file from a C++ +# file, we should do so now. Since the many flags passed to the +# compiler are usually not of much interest, we suppress the actual +# command line using the `at' sign in the first column of the rules +# and write the string indicating what we do instead. +./%.g.$(OBJEXT) : + @echo ==============debug========= $( $@ \ + || (rm -f $@ ; false) + @if test -s $@ ; then : else rm $@ ; fi + + +# To make the dependencies known to `make', we finally have to include +# them: +include Makefile.dep + + +endif # USE_CONTRIB_TRILINOS diff --git a/deal.II/examples/step-40/doc/builds-on b/deal.II/examples/step-40/doc/builds-on new file mode 100644 index 0000000000..c27eb152ec --- /dev/null +++ b/deal.II/examples/step-40/doc/builds-on @@ -0,0 +1 @@ +step-6 step-17 diff --git a/deal.II/examples/step-40/doc/intro.dox b/deal.II/examples/step-40/doc/intro.dox new file mode 100644 index 0000000000..51b95cbae4 --- /dev/null +++ b/deal.II/examples/step-40/doc/intro.dox @@ -0,0 +1,115 @@ +
    + +This program was contributed by Timo Heister, Martin Kronbichler and Wolfgang +Bangerth. +
    +This material is based upon work partly supported by the National +Science Foundation under Award No. EAR-0426271 and The California Institute of +Technology. Any opinions, findings, and conclusions or recommendations +expressed in this publication are those of the author and do not +necessarily reflect the views of the National Science Foundation or of The +California Institute of Technology. +
    + + + +

    Introduction

    + +Given today's computers, most finite element computations can be done on +a single machine. The majority of previous tutorial programs therefore +shows only this, possibly splitting up work among a number of +processors that, however, can all access the same, shared memory +space. That said, there are problems that are simply too big for a +single machine and in that case the problem has to be split up in a +suitable way among multiple machines each of which contributes its +part to the whole. A simple way to do that was shown in step-17 and +step-18, where we show how a program can use MPI to parallelize +assembling the linear system, storing it, solving it, and computing +error estimators. All of these operations scale relatively trivially, +but there was one significant drawback: for this to be moderately +simple to implement, each MPI processor had to keep its own copy of +the entire Triangulation and DoFHandler objects. Consequently, while +we can suspect (with good reasons) that the operations listed above +can scale to thousands of computers and problem sizes of billions of +cells and billions of degrees of freedom, building the one big mesh for the +entire problem these thousands of computers are solving on every last +processor is clearly not going to scale: it is going to take forever, +and maybe more importantly no single machine will have enough memory +to store a mesh that has a billion cells (at least not at the time of +writing this). In reality, programs like step-17 and step-18 can +therefore not be run on more than maybe 100 or 200 processors and even +there storing the Triangulation and DoFHandler objects consumes the +vast majority of memory on each machine. + +Consequently, we need to approach the problem differently: to scale to +very large problems each processor can only store its own little piece +of the Triangulation and DoFHandler objects. deal.II implements such a +scheme in the parallel::distributed namespace and the classes +therein. It builds on an external library, p4est (a play on the expression +parallel forest that describes the parallel storage of a +hierarchically constructed mesh as a forest of quad- or +oct-trees). You need to install and configure p4est +but apart from that all of its workings are hidden under the surface +of deal.II. + +In essence, what the parallel::distributed::Triangulation class and +associated parallel::distributed::DoFHandler objects do is to split +the global mesh so that every processor only stores a small bit it +"owns" along with one layer of "ghost" cells that surround the ones it +owns. What happens in the rest of the domain on which we want to solve +the partial differential equation is unknown to each processor and can +only be inferred through communication with other machines if such +information is needed. This implies that we also have to think about +problems in a different way than we did in, for example, step-17 and +step-18: no processor can have the entire solution vector for +postprocessing, for example, and every part of a program has to be +parallelized because no processor has all the information necessary +for sequential operations. + +A general overview of how this parallelization happens is described in +the @ref distributed documentation module. You should read it for a +top-level overview before reading through the source code of this +program. A concise discussion of many terms we will use in the program +is also provided in the @ref distributed_paper "Distributed Computing paper". +It is probably worthwhile reading it for background information on how +things work internally in this program. + + +

    The testcase

    + +This program solves essentially re-solves what we already do in +step-6, i.e. it solves the Laplace equation +@f{align*} + -\Delta u &= f \qquad &&\text{in}\ \Omega=[0,1]^2, \\ + u &= 0 \qquad &&\text{on}\ \partial\Omega. +@f} +The difference of course is now that we want to do so on a mesh that +may have a billion cells, with a billion or so degrees of +freedom. There is no doubt that doing so is completely silly for such +a simple problem, but the point of a tutorial program is, after all, +not to do something useful but to show how useful programs can be +implemented using deal.II. Be that as it may, to make things at least +a tiny bit interesting, we choose the right hand side as a +discontinuous function, +@f{align*} + f(x,y) + = + \left\{ + \begin{array}{ll} + 1 & \text{if}\ y < \frac 12 + \frac 14 \sin(4\pi x), \\ + -1 & \text{otherwise}, + \end{array} + \right. +@f} +so that the solution has a singularity along the sinusoidal line +snaking its way through the domain. As a consequence, mesh refinement +will be concentrated along this line. You can see this in the mesh +picture shown below in the results section. + +Rather than continuing here and giving a long introduction, let us go +straight to the program code. If you have read through step-6 and the +@ref distributed documentation module, most of things that are going +to happen should be familiar to you already. diff --git a/deal.II/examples/step-40/doc/kind b/deal.II/examples/step-40/doc/kind new file mode 100644 index 0000000000..c1d9154931 --- /dev/null +++ b/deal.II/examples/step-40/doc/kind @@ -0,0 +1 @@ +techniques diff --git a/deal.II/examples/step-40/doc/results.dox b/deal.II/examples/step-40/doc/results.dox new file mode 100644 index 0000000000..56bb1e36f6 --- /dev/null +++ b/deal.II/examples/step-40/doc/results.dox @@ -0,0 +1,117 @@ +

    Results

    + +DOCUMENT HOW TO RUN THE PROGRAM + +When run on a sufficiently large number of machines (say a few +thousand), this program can relatively easily solve problems with well +over one billion unknowns in less than a minute. On the other hand, +such big problems can no longer be visualized, so we also ran the +program on only 16 processors. Here are a mesh, along with its +partitioning onto the 16 processors, and the corresponding solution: + + + + + + +
    + @image html step-40.mesh.png + + @image html step-40.solution.png +
    + +The mesh on the left has a mere 7,069 cells. This is of course a +problem we would easily have been able to solve already on a single +processor using step-6, but the point of the program was to show how +to write a program that scales to many more machines. For example, +here are two graphs that show how the run time of a large number of parts +of the program scales on problems with around 52 and 375 million degrees of +freedom if we take more and more processors (these and the next couple of +graphs are taken from the @ref distributed_paper "Distributed Computing paper"): + + + + + + +
    + @image html step-40.strong2.png + + @image html step-40.strong.png +
    + +As can clearly be seen, the program scales nicely to very large +numbers of processors. The curves, in particular the linear solver, become a +bit wobble at the right end of the graphs since each processor has too little +to do to offset the cost of communication (the part of the whole problem each +processor has to solve in the above two examples is only 13,000 and 90,000 +degrees of freedom when 4,096 processors are used). + +While the strong scaling graphs above show that we can solve a problem of +fixed size faster and faster if we take more and more processors, the more +interesting question may be how big problems can become so that they can still +be solved within a reasonable time on a machine of a particular size. We show +this in the following two graphs for 256 and 4096 processors: + + + + + + +
    + @image html step-40.256.png + + @image html step-40.4096.png +
    + +What these graphs show is that all parts of the program scale linearly with +the number of degrees of freedom. This time, lines are wobbly at the left as +the size of local problems is too small. For more discussions of these results +we refer to the @ref distributed_paper "Distributed Computing paper". + +So how large are the largest problems one can solve? At the time of writing +this problem, the +limiting factor is that the program uses the BoomerAMG algebraic +multigrid method from the hypre package as +a preconditioner, which unfortunately uses signed 32-bit integers to +index the elements of a %distributed matrix. This limits the size of +problems to $2^31-1=2,147,483,647$ degrees of freedom. From the graphs +above it is obvious that the scalability would extend beyond this +number, and one could expect that given more than the 4,096 machines +shown above would also further reduce the compute time. That said, one +can certainly expect that this limit will eventually be lifted by the +hypre developers. + + + + +

    Possibilities for extensions

    + +In a sense, this program is the ultimate solver for the Laplace +equation: it can essentially solve the equation to whatever accuracy +you want, if only you have enough processors available. Since the +Laplace equation by itself is not terribly interesting at this level +of accuracy, the more interesting possibilities for extension +therefore concern not so much this program but what comes beyond +it. For example, several of the other programs in this tutorial have +significant run times, especially in 3d. It would therefore be +interesting to use the techniques explained here to extend other +programs to support parallel distributed computations. We have done +this for step-31 in the step-32 tutorial program, but the same would +apply to, for example, step-23 and step-25 for hyperbolic time +dependent problems, step-33 for gas dynamics, or step-35 for the +Navier-Stokes equations. + +Maybe equally interesting is the problem of postprocessing. As +mentioned above, we only show pictures of the solution and the mesh +for 16 processors because 4,096 processors solving 1 billion unknowns +would produce graphical output on the order of several 10 +gigabyte. Currently, no program is able to visualize this amount of +data in any reasonable way unless it also runs on at least several +hundred processors. There are, however, approaches where visualization +program directly communicate with solvers on each processor with each +visualization process rendering the part of the scene computed by the +solver on this processor. Implementing such an interface would allow +to quickly visualize things that are otherwise not amenable to +graphical display. diff --git a/deal.II/examples/step-40/doc/step-40.256.png b/deal.II/examples/step-40/doc/step-40.256.png new file mode 100644 index 0000000000..e71abac31d Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.256.png differ diff --git a/deal.II/examples/step-40/doc/step-40.4096.png b/deal.II/examples/step-40/doc/step-40.4096.png new file mode 100644 index 0000000000..c55295f02b Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.4096.png differ diff --git a/deal.II/examples/step-40/doc/step-40.mesh.png b/deal.II/examples/step-40/doc/step-40.mesh.png new file mode 100644 index 0000000000..7668eeecd5 Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.mesh.png differ diff --git a/deal.II/examples/step-40/doc/step-40.solution.png b/deal.II/examples/step-40/doc/step-40.solution.png new file mode 100644 index 0000000000..a9e261c97d Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.solution.png differ diff --git a/deal.II/examples/step-40/doc/step-40.strong.png b/deal.II/examples/step-40/doc/step-40.strong.png new file mode 100644 index 0000000000..2cae6b5db9 Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.strong.png differ diff --git a/deal.II/examples/step-40/doc/step-40.strong2.png b/deal.II/examples/step-40/doc/step-40.strong2.png new file mode 100644 index 0000000000..3b8cdbd860 Binary files /dev/null and b/deal.II/examples/step-40/doc/step-40.strong2.png differ diff --git a/deal.II/examples/step-40/doc/tooltip b/deal.II/examples/step-40/doc/tooltip new file mode 100644 index 0000000000..a8090e4c75 --- /dev/null +++ b/deal.II/examples/step-40/doc/tooltip @@ -0,0 +1 @@ +Solving the Laplace equation on adaptive meshes on thousands of processors. diff --git a/deal.II/examples/step-40/step-40-commented.cc b/deal.II/examples/step-40/step-40-commented.cc new file mode 100644 index 0000000000..b94923467c --- /dev/null +++ b/deal.II/examples/step-40/step-40-commented.cc @@ -0,0 +1,406 @@ +/* $Id$ */ +/* Author: Wolfgang Bangerth, Texas A&M University, 2009, 2010 */ +/* Timo Heister, University of Goettingen, 2009, 2010 */ + +/* $Id$ */ +/* */ +/* Copyright (C) 2009, 2010 by Timo Heister and the deal.II authors */ +/* */ +/* This file is subject to QPL and may not be distributed */ +/* without copyright and license information. Please refer */ +/* to the file deal.II/doc/license.html for the text and */ +/* further information on this license. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +using namespace dealii; + + +template +class LaplaceProblem +{ + public: + LaplaceProblem (); + ~LaplaceProblem (); + + void run (const unsigned int initial_global_refine); + + private: + void setup_system (); + void assemble_system (); + void solve (); + void refine_grid (); + void output_results (const unsigned int cycle) const; + + MPI_Comm mpi_communicator; + + parallel::distributed::Triangulation triangulation; + + DoFHandler dof_handler; + FE_Q fe; + + IndexSet locally_owned_dofs; + IndexSet locally_relevant_dofs; + + ConstraintMatrix constraints; + + PETScWrappers::MPI::SparseMatrix system_matrix; + PETScWrappers::MPI::Vector locally_relevant_solution; + PETScWrappers::MPI::Vector system_rhs; + + ConditionalOStream pcout; +}; + + + +template +LaplaceProblem::LaplaceProblem () + : + mpi_communicator (MPI_COMM_WORLD), + triangulation (mpi_communicator, + typename Triangulation::MeshSmoothing + (Triangulation::smoothing_on_refinement | + Triangulation::smoothing_on_coarsening)), + dof_handler (triangulation), + fe (2), + pcout (std::cout, + (Utilities::System:: + get_this_mpi_process(mpi_communicator) + == 0)) +{} + + + +template +LaplaceProblem::~LaplaceProblem () +{ + dof_handler.clear (); +} + + + +template +void LaplaceProblem::setup_system () +{ + locally_relevant_solution.reinit (mpi_communicator, + locally_owned_dofs, + locally_relevant_dofs); + system_rhs.reinit (mpi_communicator, + dof_handler.n_dofs(), + dof_handler.n_locally_owned_dofs()); + + locally_relevant_solution = 0; + system_rhs = 0; + + constraints.clear (); + constraints.reinit (locally_relevant_dofs); + + DoFTools::make_hanging_node_constraints (dof_handler, constraints); + VectorTools::interpolate_boundary_values (dof_handler, + 0, + ZeroFunction(), + constraints); + constraints.close (); + + + CompressedSimpleSparsityPattern csp (dof_handler.n_dofs(), + dof_handler.n_dofs(), + locally_relevant_dofs); + DoFTools::make_sparsity_pattern (dof_handler, + csp, + constraints, false); + SparsityTools::distribute_sparsity_pattern (csp, + dof_handler.n_locally_owned_dofs_per_processor(), + mpi_communicator, + locally_relevant_dofs); + system_matrix.reinit (mpi_communicator, + csp, + dof_handler.n_locally_owned_dofs_per_processor(), + dof_handler.n_locally_owned_dofs_per_processor(), + Utilities::System::get_this_mpi_process(mpi_communicator)); +} + + + +template +void LaplaceProblem::assemble_system () +{ + const QGauss quadrature_formula(3); + + FEValues fe_values (fe, quadrature_formula, + update_values | update_gradients | + update_quadrature_points | + update_JxW_values); + + const unsigned int dofs_per_cell = fe.dofs_per_cell; + const unsigned int n_q_points = quadrature_formula.size(); + + FullMatrix cell_matrix (dofs_per_cell, dofs_per_cell); + Vector cell_rhs (dofs_per_cell); + + std::vector local_dof_indices (dofs_per_cell); + + typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + for (; cell!=endc; ++cell) + if (cell->subdomain_id() == triangulation.locally_owned_subdomain()) + { + cell_matrix = 0; + cell_rhs = 0; + + fe_values.reinit (cell); + + for (unsigned int q_point=0; q_point + 0.5+0.25*sin(4.0*numbers::PI*fe_values.quadrature_point(q_point)[0]) + ? 1 : -1) * + fe_values.JxW(q_point)); + } + + cell->get_dof_indices (local_dof_indices); + constraints.distribute_local_to_global (cell_matrix, + cell_rhs, + local_dof_indices, + system_matrix, + system_rhs); + } + + system_matrix.compress (); + system_rhs.compress (); +} + + + + + +template +void LaplaceProblem::solve () +{ + PETScWrappers::MPI::Vector + completely_distributed_solution (mpi_communicator, + dof_handler.n_dofs(), + dof_handler.n_locally_owned_dofs()); + + SolverControl solver_control (dof_handler.n_dofs(), 1e-12); + + PETScWrappers::SolverCG solver(solver_control, mpi_communicator); + PETScWrappers::PreconditionBlockJacobi preconditioner(system_matrix); + + solver.solve (system_matrix, completely_distributed_solution, system_rhs, + preconditioner); + + pcout << " Solved in " << solver_control.last_step() + << " iterations." << std::endl; + + constraints.distribute (completely_distributed_solution); + + locally_relevant_solution = completely_distributed_solution; +} + + + +template +void LaplaceProblem::refine_grid () +{ + Vector estimated_error_per_cell (triangulation.n_active_cells()); + KellyErrorEstimator::estimate (dof_handler, + QGauss(3), + typename FunctionMap::type(), + locally_relevant_solution, + estimated_error_per_cell); + parallel::distributed::GridRefinement:: + refine_and_coarsen_fixed_number (triangulation, + estimated_error_per_cell, + 0.3, 0.03); + triangulation.execute_coarsening_and_refinement (); +} + + + + +template +void LaplaceProblem::output_results (const unsigned int cycle) const +{ + DataOut data_out; + data_out.attach_dof_handler (dof_handler); + data_out.add_data_vector (locally_relevant_solution, "u"); + + Vector subdomain (triangulation.n_active_cells()); + // could just fill entire vector with subdomain_id() + { + unsigned int index = 0; + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell, ++index) + subdomain(index) = (cell->is_ghost() || cell->is_artificial() + ? + -1 + : + cell->subdomain_id()); + } + data_out.add_data_vector (subdomain, "subdomain"); + data_out.build_patches (); + + const std::string filename = ("solution-" + + Utilities::int_to_string (cycle, 2) + + "." + + Utilities::int_to_string + (triangulation.locally_owned_subdomain(), 4)); + + std::ofstream output ((filename + ".vtu").c_str()); + data_out.write_vtu (output); + + if (Utilities::System::get_this_mpi_process(mpi_communicator) == 0) + { + std::vector filenames; + for (unsigned int i=0; + i +void LaplaceProblem::run (const unsigned int initial_global_refine) +{ + const unsigned int n_cycles = 12; + for (unsigned int cycle=0; cycle1) + { + refine = (unsigned int)Utilities::string_to_int(argv[1]); + } + + { + LaplaceProblem<2> laplace_problem_2d; + laplace_problem_2d.run (refine); + } + + PetscFinalize(); + } + catch (std::exception &exc) + { + std::cerr << std::endl << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Exception on processing: " << std::endl + << exc.what() << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + + return 1; + } + catch (...) + { + std::cerr << std::endl << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Unknown exception!" << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + return 1; + } + + return 0; +} diff --git a/deal.II/examples/step-40/step-40.cc b/deal.II/examples/step-40/step-40.cc new file mode 100644 index 0000000000..b8683b8024 --- /dev/null +++ b/deal.II/examples/step-40/step-40.cc @@ -0,0 +1,715 @@ +/* $Id$ */ +/* Author: Wolfgang Bangerth, Texas A&M University, 2009 */ +/* Timo Heister, University of Goettingen, 2009 */ + +/* $Id$ */ +/* */ +/* Copyright (C) 2009, 2010 by Timo Heister and the deal.II authors */ +/* */ +/* This file is subject to QPL and may not be distributed */ +/* without copyright and license information. Please refer */ +/* to the file deal.II/doc/license.html for the text and */ +/* further information on this license. */ + +// comment to use PETSc: +//#define USE_TRILINOS + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef USE_TRILINOS +#include +#include +#include +#include +#else +#include +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "timeblock.h" + +using namespace dealii; + +void print_it(Utilities::System::MinMaxAvg & result) +{ + std::cout// << "sum: " << result.sum + << " avg: " << (long)result.avg/1024 + << " min: " << (long)result.min/1024 << " @" << result.min_index + << " max: " << (long)result.max/1024 << " @" << result.max_index + << std::endl; +} + +void print_memory_stats() +{ + int myid = Utilities::System::get_this_mpi_process(MPI_COMM_WORLD); + Utilities::System::MemoryStats stats; + Utilities::System::get_memory_stats(stats); + Utilities::System::MinMaxAvg r; + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmPeak, r); + if (myid==0) + { + std::cout << "MEM: VmPeak: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmSize, r); + if (myid==0) + { + std::cout << "MEM: VmSize: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmHWM, r); + if (myid==0) + { + std::cout << "MEM: VmHWM: "; + print_it(r); + } + Utilities::System::calculate_collective_mpi_min_max_avg(MPI_COMM_WORLD, stats.VmRSS, r); + if (myid==0) + { + std::cout << "MEM: VmRSS: "; + print_it(r); + } +} + + +template +class LaplaceProblem +{ + public: + LaplaceProblem (); + ~LaplaceProblem (); + + void run (unsigned int refine); + + private: + void setup_system (); + void assemble_system (); + void solve (); + void refine_grid (); + void output_results (const unsigned int cycle) const; + + parallel::distributed::Triangulation triangulation; + + DoFHandler dof_handler; + FE_Q fe; + + IndexSet locally_owned_dofs; + IndexSet locally_relevant_dofs; + + ConstraintMatrix constraints; + +#ifdef USE_TRILINOS + TrilinosWrappers::SparseMatrix system_matrix; + TrilinosWrappers::MPI::Vector solution; + TrilinosWrappers::MPI::Vector system_rhs; + parallel::distributed::SolutionTransfer soltrans; +#else + PETScWrappers::MPI::SparseMatrix system_matrix; + PETScWrappers::MPI::Vector solution; + PETScWrappers::MPI::Vector system_rhs; + parallel::distributed::SolutionTransfer soltrans; +#endif + + ConditionalOStream pcout; + + +}; + + + +template +LaplaceProblem::LaplaceProblem () + : + triangulation (MPI_COMM_WORLD + ,typename Triangulation::MeshSmoothing (Triangulation::smoothing_on_refinement | Triangulation::smoothing_on_coarsening) + ), + dof_handler (triangulation), + fe (2), + soltrans(dof_handler), + pcout (std::cout, + (Utilities::System:: + get_this_mpi_process(MPI_COMM_WORLD) + == 0)) +{} + + + +template +LaplaceProblem::~LaplaceProblem () +{ + dof_handler.clear (); +} + + + +template +void LaplaceProblem::setup_system () +{ + TimeBlock t(pcout, "setup_system", false); + { + TimeBlock t(pcout, "_init_vectors"); +#ifdef USE_TRILINOS + solution.reinit (locally_relevant_dofs, MPI_COMM_WORLD); + system_rhs.reinit (locally_owned_dofs, MPI_COMM_WORLD); +#else + solution.reinit ( MPI_COMM_WORLD, locally_owned_dofs, locally_relevant_dofs); + system_rhs.reinit (MPI_COMM_WORLD, dof_handler.n_dofs(), + dof_handler.n_locally_owned_dofs()); + + solution = 0; + system_rhs = 0; +#endif + } + + constraints.clear (); + constraints.reinit (locally_relevant_dofs); + + { + + TimeBlock t(pcout, "_make_hn"); + + DoFTools::make_hanging_node_constraints (static_cast&>(dof_handler), + constraints); + } + + { + TimeBlock t(pcout, "_interpol_bv"); + VectorTools::interpolate_boundary_values (static_cast&>(dof_handler), + 0, + ZeroFunction(), + constraints); + constraints.close (); + } + + + CompressedSimpleSparsityPattern csp (dof_handler.n_dofs(), + dof_handler.n_dofs(), + locally_relevant_dofs); + { + TimeBlock t(pcout, "_make_sp", false); + + DoFTools::make_sparsity_pattern (static_cast&>(dof_handler), + csp, + constraints, false); + } + + { + TimeBlock t(pcout, "_init_matrix", false); +#ifdef USE_TRILINOS + system_matrix.reinit (locally_owned_dofs, csp, MPI_COMM_WORLD, true); +#else + + { + TimeBlock t(pcout, "__prep_csp", false); + SparsityTools::distribute_sparsity_pattern<>(csp, + dof_handler.n_locally_owned_dofs_per_processor(), + MPI_COMM_WORLD, + locally_relevant_dofs); + } + + system_matrix.reinit (MPI_COMM_WORLD, + csp, + dof_handler.n_locally_owned_dofs_per_processor(), + dof_handler.n_locally_owned_dofs_per_processor(), + Utilities::System::get_this_mpi_process(MPI_COMM_WORLD)); +#if (PETSC_VERSION_MAJOR <= 2) +// MatSetOption (system_matrix, MAT_YES_NEW_NONZERO_LOCATIONS); +#else +// MatSetOption (system_matrix, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE); +// MatSetOption (system_matrix, MAT_NEW_NONZERO_LOCATIONS, PETSC_TRUE); +#endif + +#endif + } +} + + + +template +void LaplaceProblem::assemble_system () +{ + TimeBlock t(pcout, "assemble"); + + const QGauss quadrature_formula(3); + + FEValues fe_values (fe, quadrature_formula, + update_values | update_gradients | + update_quadrature_points | + update_JxW_values); + + const unsigned int dofs_per_cell = fe.dofs_per_cell; + const unsigned int n_q_points = quadrature_formula.size(); + + FullMatrix cell_matrix (dofs_per_cell, dofs_per_cell); + Vector cell_rhs (dofs_per_cell); + + std::vector local_dof_indices (dofs_per_cell); + + typename DoFHandler::active_cell_iterator + cell = dof_handler.begin_active(), + endc = dof_handler.end(); + for (; cell!=endc; ++cell) + if (cell->subdomain_id() == triangulation.locally_owned_subdomain()) + { + cell_matrix = 0; + cell_rhs = 0; + + fe_values.reinit (cell); + + for (unsigned int q_point=0; q_pointget_dof_indices (local_dof_indices); + + constraints.distribute_local_to_global (cell_matrix, + cell_rhs, + local_dof_indices, + system_matrix, + system_rhs); + } + +#ifdef USE_TRILINOS + system_rhs.compress (Add); +#else + system_rhs.compress (); +#endif + system_matrix.compress (); +} + + + + + +template +void LaplaceProblem::solve () +{ + // we do not want to get spammed with solver info. + deallog.depth_console (0); + + TimeBlock t(pcout, "solve", false); + +#ifdef USE_TRILINOS + TrilinosWrappers::MPI::Vector distributed_solution (locally_owned_dofs, + MPI_COMM_WORLD); + + distributed_solution.reinit(solution, false, true); +#else + + PETScWrappers::MPI::Vector distributed_solution(MPI_COMM_WORLD, + dof_handler.n_dofs(), + dof_handler.n_locally_owned_dofs()); + + distributed_solution = solution; +#endif + + print_memory_stats(); + + SolverControl solver_control (solution.size(), 1e-12); +#ifdef USE_TRILINOS + SolverCG solver (solver_control); + + TrilinosWrappers::PreconditionAMG preconditioner; + preconditioner.initialize(system_matrix); +#else + PETScWrappers::SolverCG solver(solver_control, MPI_COMM_WORLD); + PETScWrappers::PreconditionBlockJacobi preconditioner(system_matrix); + +#endif + + { // memory consumption + pcout << "Mem: Tria (p4est) DofH Constraints Mat X Rhs I_lo I_lr" + << std::endl; +// MPI_Barrier(MPI_COMM_WORLD); + + pcout << "MEM: " + << " " << triangulation.memory_consumption() + << " " << triangulation.memory_consumption_p4est() + << " " << dof_handler.memory_consumption() + << " " << constraints.memory_consumption() + << " " << system_matrix.memory_consumption() + << " " << solution.memory_consumption() + << " " << system_rhs.memory_consumption() + << " " << locally_owned_dofs.memory_consumption() + << " " << locally_relevant_dofs.memory_consumption() +// << " " << preconditioner.memory_consumption() + << " ~sum=" << (triangulation.memory_consumption() + dof_handler.memory_consumption() + + constraints.memory_consumption() + system_matrix.memory_consumption() + + solution.memory_consumption() + system_rhs.memory_consumption() + + locally_owned_dofs.memory_consumption() + + locally_relevant_dofs.memory_consumption() )/1024 + << " kB" + << std::endl; + +// MPI_Barrier(MPI_COMM_WORLD); + } + + + solver.solve (system_matrix, distributed_solution, system_rhs, + preconditioner); + + print_memory_stats(); + + pcout << " Solved in " << solver_control.last_step() + << " iterations." << std::endl; + + constraints.distribute (distributed_solution); + solution = distributed_solution; + +#ifndef USE_TRILINOS + solution.update_ghost_values(); +#endif + + deallog.depth_console (1); +} + + + +template +void LaplaceProblem::refine_grid () +{ + TimerOutput computing_timer(pcout, TimerOutput::summary, + TimerOutput::wall_times); + + computing_timer.enter_section ("Estimating"); + + Vector estimated_error_per_cell (triangulation.n_active_cells()); + + { + + TimeBlock t(pcout, "kelly"); + + KellyErrorEstimator::estimate (static_cast&>(dof_handler), + QGauss(3), + typename FunctionMap::type(), + solution, + estimated_error_per_cell); + } + + computing_timer.exit_section(); + + computing_timer.enter_section ("Marking"); + { + TimeBlock t(pcout, "marking"); + + parallel::distributed::GridRefinement:: + refine_and_coarsen_fixed_number (triangulation, + estimated_error_per_cell, + 0.3, 0.03); + } + + computing_timer.exit_section(); + + computing_timer.enter_section ("Refining"); + + + { + TimeBlock t(pcout, "prep_c&r"); + + triangulation.prepare_coarsening_and_refinement(); + } + + if (0) + { + //output refinement information for + //debugging + + const std::string filename = ("ref." + + Utilities::int_to_string + (triangulation.locally_owned_subdomain(), 4) + + ".vtk"); + std::ofstream output (filename.c_str()); + + DataOut data_out; + data_out.attach_dof_handler (dof_handler); + + unsigned int n_coarse=0; + Vector subdomain (triangulation.n_active_cells()); + { + unsigned int index = 0; + + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell, ++index) + { + subdomain(index)=0; + + if (cell->is_ghost() || cell->is_artificial()) + subdomain(index)=-4; + + if (cell->refine_flag_set()) + subdomain(index)+=1; + if (cell->coarsen_flag_set()) + { + subdomain(index)+=2; + ++n_coarse; + } + } + } + std::cout << "id=" << triangulation.locally_owned_subdomain() << " n_coarsen=" << n_coarse << std::endl; + + data_out.add_data_vector (subdomain, "info"); + data_out.build_patches (); + data_out.write_vtk (output); + } + + { + TimeBlock t(pcout, "soltrans.prep"); + + soltrans.prepare_for_coarsening_and_refinement(solution); + } + + { + TimeBlock t(pcout, "c&r", false); + + triangulation.execute_coarsening_and_refinement (); + } + + computing_timer.exit_section(); +} + + + + +template +void LaplaceProblem::output_results (const unsigned int cycle) const +{ + const std::string filename = ("solution-" + + Utilities::int_to_string (cycle, 2) + + "." + + Utilities::int_to_string + (triangulation.locally_owned_subdomain(), 4) + + ".d2"); + std::ofstream output (filename.c_str()); + + DataOut data_out; + data_out.attach_dof_handler (dof_handler); + data_out.add_data_vector (solution, "u"); + + Vector subdomain (triangulation.n_active_cells()); + { + unsigned int index = 0; + for (typename Triangulation::active_cell_iterator + cell = triangulation.begin_active(); + cell != triangulation.end(); ++cell, ++index) + subdomain(index) = (cell->is_ghost() || cell->is_artificial() + ? + -1 + : + cell->subdomain_id()); + } + data_out.add_data_vector (subdomain, "subdomain"); + + data_out.build_patches (); + + data_out.write_deal_II_intermediate (output); +} + + + +template +void LaplaceProblem::run (unsigned int refine) +{ + { + TimeBlock t(pcout, "empty"); + } + + unsigned int cycles = 12; + + pcout << "running " << cycles << " cycles with refine= " << refine << +#ifdef USE_TRILINOS + " with Trilinos" +#else + " with PETSc" +#endif + << std::endl; + + for (unsigned int cycle=0; cycle t(pcout, "dist_dofs"); + dof_handler.distribute_dofs (fe); + } + + + locally_owned_dofs = dof_handler.locally_owned_dofs (); + DoFTools::extract_locally_relevant_dofs (dof_handler, + locally_relevant_dofs); + + pcout << " Number of degrees of freedom: " + << dof_handler.n_dofs() + << std::endl; +/* pcout << " ("; + for (unsigned int i=0; i0) + { + TimeBlock t(pcout, "soltrans.interp"); + + soltrans.interpolate(solution); + } + + + computing_timer.exit_section(); + + computing_timer.enter_section ("Assembling system"); + assemble_system (); + computing_timer.exit_section(); + + computing_timer.enter_section ("Solving system"); + solve (); + computing_timer.exit_section(); + +// output_results (cycle); + + pcout << std::endl; + } +} + + + +int main(int argc, char *argv[]) +{ + try + { +#ifdef USE_TRILINOS + Utilities::System::MPI_InitFinalize mpi_initialization(argc, argv); +#else + PetscInitialize(&argc, &argv); +#endif + + deallog.depth_console (1); + + int refine=5; + if (argc>1) + { + refine = (unsigned int)Utilities::string_to_int(argv[1]); + } + + { + LaplaceProblem<2> laplace_problem_2d; + laplace_problem_2d.run (refine); + } + + print_memory_stats(); + +#ifndef USE_TRILINOS + PetscLogPrintSummary(MPI_COMM_WORLD, "petsc.log" ); + PetscFinalize(); +#endif + + } + catch (std::exception &exc) + { + std::cerr << std::endl << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Exception on processing: " << std::endl + << exc.what() << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + + return 1; + } + catch (...) + { + std::cerr << std::endl << std::endl + << "----------------------------------------------------" + << std::endl; + std::cerr << "Unknown exception!" << std::endl + << "Aborting!" << std::endl + << "----------------------------------------------------" + << std::endl; + return 1; + } + + return 0; +} diff --git a/deal.II/examples/step-40/timeblock.h b/deal.II/examples/step-40/timeblock.h new file mode 100644 index 0000000000..d86eff8ae0 --- /dev/null +++ b/deal.II/examples/step-40/timeblock.h @@ -0,0 +1,54 @@ +/** + * copyright: + * Research Group on Numerical Methods for PDEs + * University of Göttingen + * + * lube@math.uni-goettingen.de + */ + +#ifndef TIMEBLOCK_H +#define TIMEBLOCK_H + +template +class TimeBlock +{ + public: + TimeBlock(STREAM & stream_, const char* blockname, bool singleline=true) + : m_blockname(blockname), + m_singleline(singleline), + m_timer(MPI_COMM_WORLD, true), + stream(stream_) + { + MPI_Barrier(MPI_COMM_WORLD); + stream << blockname << " ... "; + if (singleline) + stream << std::flush; + else + stream << std::endl; + m_timer.start(); + } + + ~TimeBlock() + { + m_timer.stop(); + + if (!m_singleline) + stream << m_blockname << " took "; + + stream << std::fixed << std::setprecision(4); + + m_timer.print_data(stream); + + MPI_Barrier(MPI_COMM_WORLD); + } + + + private: + const char* m_blockname; + bool m_singleline; + bool m_issection; + dealii::Timer m_timer; + STREAM & stream; +}; + +#endif diff --git a/deal.II/lac/include/lac/block_matrix_base.h b/deal.II/lac/include/lac/block_matrix_base.h index bdda476633..4da101c56a 100644 --- a/deal.II/lac/include/lac/block_matrix_base.h +++ b/deal.II/lac/include/lac/block_matrix_base.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -898,6 +899,16 @@ class BlockMatrixBase : public Subscriptor */ const BlockIndices & get_column_indices () const; + /** + * Determine an estimate for the memory + * consumption (in bytes) of this + * object. Note that only the memory + * reserved on the current processor is + * returned in case this is called in + * an MPI-based program. + */ + unsigned int memory_consumption () const; + /** @addtogroup Exceptions * @{ */ @@ -1710,6 +1721,29 @@ copy_from (const BlockMatrixType &source) } +template +unsigned int +BlockMatrixBase::memory_consumption () const +{ + unsigned int mem = + MemoryConsumption::memory_consumption(row_block_indices)+ + MemoryConsumption::memory_consumption(column_block_indices)+ + MemoryConsumption::memory_consumption(sub_objects)+ + MemoryConsumption::memory_consumption(counter_within_block)+ + MemoryConsumption::memory_consumption(column_indices)+ + MemoryConsumption::memory_consumption(column_values); + + for (unsigned int r=0; rsub_objects[r][c]; + mem += MemoryConsumption::memory_consumption(*p); + } + + return mem; +} + + template inline diff --git a/deal.II/lac/include/lac/block_sparsity_pattern.h b/deal.II/lac/include/lac/block_sparsity_pattern.h index 29fbdb146c..3a4612e954 100644 --- a/deal.II/lac/include/lac/block_sparsity_pattern.h +++ b/deal.II/lac/include/lac/block_sparsity_pattern.h @@ -916,12 +916,26 @@ class BlockCompressedSimpleSparsityPattern : public BlockSparsityPatternBase& row_block_sizes, const std::vector& col_block_sizes); - + /** - * Resize the matrix to a tensor - * product of matrices with - * dimensions defined by the - * arguments. + * Initialize the pattern with symmetric + * blocks. The number of IndexSets in the + * vector determine the number of rows + * and columns of blocks. The size of + * each block is determined by the size() + * of the respective IndexSet. Each block + * only stores the rows given by the + * values in the IndexSet, which is + * useful for distributed memory parallel + * computations and usually corresponds + * to the locally owned DoFs. + */ + BlockCompressedSimpleSparsityPattern (const std::vector & partitioning); + + /** + * Resize the pattern to a tensor product + * of matrices with dimensions defined by + * the arguments. * * The matrix will have as many * block rows and columns as @@ -935,6 +949,15 @@ class BlockCompressedSimpleSparsityPattern : public BlockSparsityPatternBase &row_block_sizes, const std::vector< unsigned int > &col_block_sizes); + /** + * Resize the pattern with symmetric + * blocks determined by the size() of + * each IndexSet. See the constructor + * taking a vector of IndexSets for + * details. + */ + void reinit(const std::vector & partitioning); + /** * Allow the use of the reinit * functions of the base class as @@ -1030,17 +1053,14 @@ namespace TrilinosWrappers /** * Initialize the pattern with an array - * of index sets that specifies both - * rows and columns of the matrix (so - * the final matrix will be a square - * matrix), where the IndexSet - * specifies the parallel distribution - * of the degrees of freedom on the - * individual block. This function is - * equivalent to calling the second - * constructor with the length of the - * mapping vector and then entering the - * index values. + * of index sets that specifies both rows + * and columns of the matrix (so the + * final matrix will be a square matrix), + * where the size() of the IndexSets + * specifies the size of the blocks and + * the values in each IndexSet denotes + * the rows that are going to be saved in + * each block. */ BlockSparsityPattern (const std::vector& parallel_partitioning, const MPI_Comm &communicator = MPI_COMM_WORLD); @@ -1074,10 +1094,9 @@ namespace TrilinosWrappers /** * Resize the matrix to a square tensor - * product of matrices with parallel - * distribution according to the - * specifications in the array of - * Epetra_Maps. + * product of matrices. See the + * constructor that takes a vector of + * IndexSets for details. */ void reinit (const std::vector& parallel_partitioning, const MPI_Comm & communicator = MPI_COMM_WORLD); diff --git a/deal.II/lac/include/lac/constraint_matrix.h b/deal.II/lac/include/lac/constraint_matrix.h index 33f4ab8a68..eb03db3550 100644 --- a/deal.II/lac/include/lac/constraint_matrix.h +++ b/deal.II/lac/include/lac/constraint_matrix.h @@ -243,11 +243,9 @@ class ConstraintMatrix : public Subscriptor */ /** - * Add a new line to the - * matrix. If the line already - * exists, then the function - * simply returns without doing - * anything. + * Add a new line to the matrix. If the + * line already exists, then the function + * simply returns without doing anything. */ void add_line (const unsigned int line); @@ -1371,6 +1369,10 @@ class ConstraintMatrix : public Subscriptor * or Trilinos vector wrapper class, or * any other type having the same * interface. + * + * Note that if called with a @p + * TrilinosWrappers::MPI::Vector it may + * not contain ghost elements. */ template void distribute (VectorType &vec) const; diff --git a/deal.II/lac/include/lac/petsc_parallel_vector.h b/deal.II/lac/include/lac/petsc_parallel_vector.h index bfec7d151a..5fe82f7319 100644 --- a/deal.II/lac/include/lac/petsc_parallel_vector.h +++ b/deal.II/lac/include/lac/petsc_parallel_vector.h @@ -187,7 +187,7 @@ namespace PETScWrappers explicit Vector (const MPI_Comm &communicator, const unsigned int n, const unsigned int local_size); - + /** * Copy-constructor from deal.II @@ -229,7 +229,7 @@ namespace PETScWrappers const unsigned int local_size); - /** + /** * Constructs a new parallel PETSc * vector from an Indexset. Note that * @p local must be contiguous and @@ -375,7 +375,7 @@ namespace PETScWrappers void reinit (const Vector &v, const bool fast = false); - /** + /** * Reinit as a ghosted vector. See * constructor with same signature * for more details. diff --git a/deal.II/lac/include/lac/petsc_vector_base.h b/deal.II/lac/include/lac/petsc_vector_base.h index 0b93a15330..9870b3a289 100644 --- a/deal.II/lac/include/lac/petsc_vector_base.h +++ b/deal.II/lac/include/lac/petsc_vector_base.h @@ -664,7 +664,7 @@ namespace PETScWrappers void ratio (const VectorBase &a, const VectorBase &b); - /** + /** * Updates the ghost values of this * vector. This is necessary after any * modification before reading ghost @@ -753,7 +753,7 @@ namespace PETScWrappers */ bool ghosted; - /** + /** * This vector contains the global * indices of the ghost values. The * location in this vector denotes the diff --git a/deal.II/lac/include/lac/sparsity_tools.h b/deal.II/lac/include/lac/sparsity_tools.h index dabf7e35e6..32225be219 100644 --- a/deal.II/lac/include/lac/sparsity_tools.h +++ b/deal.II/lac/include/lac/sparsity_tools.h @@ -199,21 +199,42 @@ namespace SparsityTools #ifdef DEAL_II_COMPILER_SUPPORTS_MPI /** * Communciate rows in a compressed - * sparsity pattern over MPI. The parameter - * @p csp is modified inline. All entries - * in rows that belong to a different - * processor are send to them and added - * there. The ownership is determined by - * the parameter @p rows_per_cpu. The - * IndexSet @p myrange should be the one - * used in the constructor of the + * sparsity pattern over MPI. + * + * @param csp is the sparsity + * pattern that has been built + * locally and for which we need to + * exchange entries with other + * processors to make sure that + * each processor knows all the + * elements of the rows of a matrix + * it stores and that may + * eventually be written to. This + * sparsity pattern will be changes + * as a result of this function: + * All entries in rows that belong + * to a different processor are + * sent to them and added there. + * + * @param rows_per_cpu determines ownership of rows. + * + * @param mpi_comm is the MPI + * communicator that is shared + * between the processors that all + * participate in this operation. + * + * @param myrange indicates the + * range of elements stored locally + * and should be the one used in + * the constructor of the * CompressedSimpleSparsityPattern. Only - * the rows in @p csp contained in @p - * myrange are checked for transfer. This - * function needs to be used with - * PETScWrappers::MPI::SparseMatrix for it - * to work correctly in a parallel - * calculation. + * rows contained in myrange are + * checked in csp for transfer. + * This function needs to be used + * with + * PETScWrappers::MPI::SparseMatrix + * for it to work correctly in a + * parallel computation. */ template void distribute_sparsity_pattern(CSP_t & csp, @@ -222,7 +243,7 @@ namespace SparsityTools const IndexSet & myrange); #endif - + /** * Exception */ diff --git a/deal.II/lac/include/lac/trilinos_sparse_matrix.h b/deal.II/lac/include/lac/trilinos_sparse_matrix.h index b5428cae27..569bcdbaa7 100644 --- a/deal.II/lac/include/lac/trilinos_sparse_matrix.h +++ b/deal.II/lac/include/lac/trilinos_sparse_matrix.h @@ -3231,7 +3231,9 @@ namespace TrilinosWrappers // the content -- this is what we need // since we're going to overwrite that // anyway in the vmult operation. - temp_vector.reinit(dst, true); + // TODO[TH]: workaround, because fast is + // not reliable with MPI right now. + temp_vector.reinit(dst, false); vmult (temp_vector, src); dst += temp_vector; diff --git a/deal.II/lac/include/lac/trilinos_vector.h b/deal.II/lac/include/lac/trilinos_vector.h index 8e730fe177..551ace6446 100644 --- a/deal.II/lac/include/lac/trilinos_vector.h +++ b/deal.II/lac/include/lac/trilinos_vector.h @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2008, 2009 by the deal.II authors +// Copyright (C) 2008, 2009, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -68,7 +68,7 @@ namespace TrilinosWrappers * to avoid using the wrong data, you need to call Vector::compress() * before you actually use the vectors. * - *

    Parallel communication model

    + *

    Parallel communication model

    * * The parallel functionality of Trilinos is built on top of the Message * Passing Interface (MPI). MPI's communication model is built on diff --git a/deal.II/lac/include/lac/trilinos_vector_base.h b/deal.II/lac/include/lac/trilinos_vector_base.h index adcfaa6410..21d13c78ff 100644 --- a/deal.II/lac/include/lac/trilinos_vector_base.h +++ b/deal.II/lac/include/lac/trilinos_vector_base.h @@ -18,6 +18,7 @@ #ifdef DEAL_II_USE_TRILINOS +#include # include # include # include @@ -896,8 +897,7 @@ namespace TrilinosWrappers /** * Estimate for the memory - * consumption (not implemented - * for this class). + * consumption in bytes. */ unsigned int memory_consumption () const; //@} @@ -1132,11 +1132,29 @@ namespace TrilinosWrappers void VectorBase::compress (const Epetra_CombineMode given_last_action) { + Epetra_CombineMode mode = (last_action != Zero) ? + last_action : given_last_action; + #ifdef DEBUG + #ifdef DEAL_II_COMPILER_SUPPORTS_MPI + // check that every process has decided + // to use the same mode. This will + // otherwise result in undefined + // behaviour in the call to + // GlobalAssemble(). + double double_mode = mode; + struct Utilities::System::MinMaxAvg result; + Utilities::System::calculate_collective_mpi_min_max_avg( + dynamic_cast(&vector_partitioner().Comm())->GetMpiComm(), + double_mode, + result); + Assert(result.max-result.min<1e-5, ExcTrilinosError(0)); + + #endif + #endif + // Now pass over the information about // what we did last to the vector. - const int ierr = vector->GlobalAssemble(last_action != Zero ? - last_action : - given_last_action); + const int ierr = vector->GlobalAssemble(mode); AssertThrow (ierr == 0, ExcTrilinosError(ierr)); last_action = Zero; @@ -1193,6 +1211,11 @@ namespace TrilinosWrappers const unsigned int *indices, const TrilinosScalar *values) { + // if we have ghost values, do not allow + // writing to this vector at all. + Assert (vector->Map().UniqueGIDs()==true, + ExcGhostsPresent()); + if (last_action == Add) vector->GlobalAssemble(Add); @@ -1250,6 +1273,11 @@ namespace TrilinosWrappers const unsigned int *indices, const TrilinosScalar *values) { + // if we have ghost values, do not allow + // writing to this vector at all. + Assert (vector->Map().UniqueGIDs()==true, + ExcGhostsPresent()); + if (last_action != Add) { if (last_action == Insert) diff --git a/deal.II/lac/source/block_sparsity_pattern.cc b/deal.II/lac/source/block_sparsity_pattern.cc index 50e3c59605..55eeb571a9 100644 --- a/deal.II/lac/source/block_sparsity_pattern.cc +++ b/deal.II/lac/source/block_sparsity_pattern.cc @@ -265,6 +265,33 @@ BlockSparsityPatternBase::print(std::ostream& out) const } +template <> +void +BlockSparsityPatternBase::print(std::ostream& out) const +{ + unsigned int k=0; + for (unsigned int ib=0;ib void BlockSparsityPatternBase::print_gnuplot(std::ostream &out) const @@ -598,6 +625,21 @@ BlockCompressedSimpleSparsityPattern (const std::vector& row_indic } +BlockCompressedSimpleSparsityPattern:: +BlockCompressedSimpleSparsityPattern (const std::vector & partitioning) + : + BlockSparsityPatternBase(partitioning.size(), + partitioning.size()) +{ + for (unsigned int i=0;iblock(i,j).reinit(partitioning[i].size(), + partitioning[j].size(), + partitioning[i]); + this->collect_sizes(); +} + + void BlockCompressedSimpleSparsityPattern::reinit ( @@ -612,6 +654,19 @@ BlockCompressedSimpleSparsityPattern::reinit ( this->collect_sizes(); } +void +BlockCompressedSimpleSparsityPattern::reinit ( + const std::vector< IndexSet > & partitioning) +{ + BlockSparsityPatternBase:: + reinit(partitioning.size(), partitioning.size()); + for (unsigned int i=0;iblock(i,j).reinit(partitioning[i].size(), + partitioning[j].size(), + partitioning[i]); + this->collect_sizes(); +} #ifdef DEAL_II_USE_TRILINOS diff --git a/deal.II/lac/source/compressed_simple_sparsity_pattern.cc b/deal.II/lac/source/compressed_simple_sparsity_pattern.cc index db3404a9e2..ede39e6b38 100644 --- a/deal.II/lac/source/compressed_simple_sparsity_pattern.cc +++ b/deal.II/lac/source/compressed_simple_sparsity_pattern.cc @@ -279,6 +279,8 @@ CompressedSimpleSparsityPattern::reinit (const unsigned int m, cols = n; rowset=rowset_; + Assert(rowset.size()==0 || rowset.size() == m, ExcInvalidConstructorCall()); + std::vector new_lines (rowset.size()==0 ? rows : rowset.n_elements()); lines.swap (new_lines); } diff --git a/deal.II/lac/source/constraint_matrix.cc b/deal.II/lac/source/constraint_matrix.cc index ea5f067941..cc6aa46a28 100644 --- a/deal.II/lac/source/constraint_matrix.cc +++ b/deal.II/lac/source/constraint_matrix.cc @@ -1924,7 +1924,11 @@ ConstraintMatrix::distribute (TrilinosWrappers::MPI::Vector &vec) const vec(it->line) = new_value; } - vec.compress (); + // some processes might not apply + // constraints, so we need to explicitly + // state, that the others are doing an + // insert here: + vec.compress (Insert); } @@ -2018,6 +2022,9 @@ ConstraintMatrix::distribute (TrilinosWrappers::MPI::BlockVector &vec) const } } + // force every processor to write something + unsigned int idx = vec.block(0).local_range().first; + vec(idx) = vec(idx); vec.compress (); } diff --git a/deal.II/lac/source/petsc_parallel_sparse_matrix.cc b/deal.II/lac/source/petsc_parallel_sparse_matrix.cc index ac4163e11d..5fa947c3c7 100644 --- a/deal.II/lac/source/petsc_parallel_sparse_matrix.cc +++ b/deal.II/lac/source/petsc_parallel_sparse_matrix.cc @@ -310,8 +310,8 @@ namespace PETScWrappers //see below) const unsigned int - local_col_end = local_col_start + local_columns_per_process[this_process]; - + local_col_end = local_col_start + local_columns_per_process[this_process]; + // then count the elements in- and // out-of-window for the rows we own #ifdef PETSC_USE_64BIT_INDICES @@ -351,7 +351,7 @@ namespace PETScWrappers 0, &row_lengths_out_of_window[0], &matrix); AssertThrow (ierr == 0, ExcPETScError(ierr)); - + #else //PETSC_VERSION>=2.3.3 // new version to create the matrix. We // do not set row length but set the @@ -449,7 +449,7 @@ namespace PETScWrappers // now copy over the information // from the sparsity pattern. - { + { #ifdef PETSC_USE_64BIT_INDICES PetscInt #else @@ -465,9 +465,9 @@ namespace PETScWrappers std::copy(row_start, row_end, ptr); ptr += row_end - row_start; - } + } } - + // then call the petsc function // that summarily allocates these @@ -481,7 +481,7 @@ namespace PETScWrappers // this is only needed for old // PETSc versions: - // for some reason, it does not + // for some reason, it does not // seem to be possible to force // actual allocation of actual // entries by using the last @@ -539,7 +539,7 @@ namespace PETScWrappers #else ierr = MatSetOption (matrix, MAT_NO_NEW_NONZERO_LOCATIONS); AssertThrow (ierr == 0, ExcPETScError(ierr)); -#endif +#endif #else int ierr; #ifdef DEBUG diff --git a/deal.II/lac/source/petsc_parallel_vector.cc b/deal.II/lac/source/petsc_parallel_vector.cc index 78701fc1c3..3d62879ed7 100644 --- a/deal.II/lac/source/petsc_parallel_vector.cc +++ b/deal.II/lac/source/petsc_parallel_vector.cc @@ -63,7 +63,7 @@ namespace PETScWrappers VectorBase::operator = (v); } - + Vector::Vector (const MPI_Comm &communicator, const IndexSet & local, @@ -137,7 +137,7 @@ namespace PETScWrappers } - + void Vector::reinit (const MPI_Comm &comm, const IndexSet & local, @@ -271,11 +271,11 @@ namespace PETScWrappers #endif - } + } -} + } } diff --git a/deal.II/lac/source/petsc_vector_base.cc b/deal.II/lac/source/petsc_vector_base.cc index 336483ccbd..4121698551 100644 --- a/deal.II/lac/source/petsc_vector_base.cc +++ b/deal.II/lac/source/petsc_vector_base.cc @@ -16,6 +16,7 @@ #ifdef DEAL_II_USE_PETSC +# include # include # include # include @@ -43,7 +44,7 @@ namespace PETScWrappers if (dynamic_cast(&vector) != 0) { #if (PETSC_VERSION_MAJOR <= 2) && (PETSC_VERSION_MINOR < 3) - PetscScalar *ptr; + PetscScalar *ptr; int ierr = VecGetArray (vector.vector, &ptr); AssertThrow (ierr == 0, ExcPETScError(ierr)); @@ -1053,12 +1054,24 @@ namespace PETScWrappers } - unsigned int VectorBase::memory_consumption () const - { - AssertThrow(false, ExcNotImplemented() ); - return 0; + { + unsigned int mem = sizeof(Vec)+sizeof(LastAction::Values) + +MemoryConsumption::memory_consumption(ghosted) + +MemoryConsumption::memory_consumption(ghost_indices); + + // TH: I am relatively sure that PETSc is + // storing the local data in a contigious + // block without indices: + mem += local_size()*sizeof(PetscScalar); + // assume that PETSc is storing one index + // and one double per ghost element + if (ghosted) + mem += ghost_indices.n_elements()*(sizeof(PetscScalar)+sizeof(int)); + + //TODO[TH]: size of constant memory for PETSc? + return mem; } @@ -1123,8 +1136,8 @@ namespace PETScWrappers AssertThrow (ierr == 0, ExcPETScError(ierr)); ierr = VecGhostUpdateEnd(vector, INSERT_VALUES, SCATTER_FORWARD); AssertThrow (ierr == 0, ExcPETScError(ierr)); -} - + } + } diff --git a/deal.II/lac/source/sparsity_tools.cc b/deal.II/lac/source/sparsity_tools.cc index 7dc9f41f26..2aea7cb203 100644 --- a/deal.II/lac/source/sparsity_tools.cc +++ b/deal.II/lac/source/sparsity_tools.cc @@ -448,7 +448,7 @@ namespace SparsityTools { row_idx+=rows_per_cpu[myid]-1; continue; -} + } unsigned int rlen = csp.row_length(row); diff --git a/deal.II/lac/source/trilinos_vector_base.cc b/deal.II/lac/source/trilinos_vector_base.cc index 3ec70b5fe9..da2469e31b 100644 --- a/deal.II/lac/source/trilinos_vector_base.cc +++ b/deal.II/lac/source/trilinos_vector_base.cc @@ -2,7 +2,7 @@ // $Id$ // Version: $Name$ // -// Copyright (C) 2008 by the deal.II authors +// Copyright (C) 2008, 2010 by the deal.II authors // // This file is subject to QPL and may not be distributed // without copyright and license information. Please refer @@ -12,6 +12,7 @@ //--------------------------------------------------------------------------- +#include #include #ifdef DEAL_II_USE_TRILINOS @@ -397,8 +398,13 @@ namespace TrilinosWrappers unsigned int VectorBase::memory_consumption () const { - AssertThrow(false, ExcNotImplemented() ); - return 0; + //TODO[TH]: No accurate memory + //consumption for Trilinos vectors + //yet. This is a rough approximation with + //one index and the value per local + //entry. + return sizeof(*this) + + this->local_size()*( sizeof(double)+sizeof(int) ); } } /* end of namespace TrilinosWrappers */