]> https://gitweb.dealii.org/ - dealii.git/commitdiff
Merge branch_threading_building_blocks.
authorWolfgang Bangerth <bangerth@math.tamu.edu>
Fri, 15 May 2009 17:59:46 +0000 (17:59 +0000)
committerWolfgang Bangerth <bangerth@math.tamu.edu>
Fri, 15 May 2009 17:59:46 +0000 (17:59 +0000)
git-svn-id: https://svn.dealii.org/trunk@18849 0785d39b-7218-0410-832d-ea1e28bc413d

75 files changed:
deal.II/Makefile
deal.II/aclocal.m4
deal.II/base/Makefile
deal.II/base/include/base/config.h.in
deal.II/base/include/base/parallel.h [new file with mode: 0644]
deal.II/base/include/base/template_constraints.h
deal.II/base/include/base/thread_management.h
deal.II/base/include/base/work_stream.h [new file with mode: 0644]
deal.II/base/source/data_out_base.cc
deal.II/base/source/quadrature_lib.cc
deal.II/base/source/thread_management.cc
deal.II/common/Make.global_options.in
deal.II/common/Make.rules
deal.II/configure
deal.II/configure.in
deal.II/contrib/Makefile.in
deal.II/contrib/functionparser/Makefile
deal.II/deal.II/Makefile
deal.II/deal.II/include/dofs/dof_accessor.templates.h
deal.II/deal.II/include/grid/filtered_iterator.h
deal.II/deal.II/include/hp/fe_values.h
deal.II/deal.II/include/numerics/data_out.h
deal.II/deal.II/include/numerics/data_out_faces.h
deal.II/deal.II/include/numerics/data_out_rotation.h
deal.II/deal.II/include/numerics/error_estimator.h
deal.II/deal.II/include/numerics/matrices.h
deal.II/deal.II/include/numerics/vectors.templates.h
deal.II/deal.II/source/dofs/dof_renumbering.cc
deal.II/deal.II/source/dofs/dof_tools.cc
deal.II/deal.II/source/grid/grid_generator.cc
deal.II/deal.II/source/multigrid/mg_dof_accessor.cc
deal.II/deal.II/source/multigrid/mg_dof_tools.cc
deal.II/deal.II/source/numerics/data_out.cc
deal.II/deal.II/source/numerics/data_out_faces.cc
deal.II/deal.II/source/numerics/data_out_rotation.cc
deal.II/deal.II/source/numerics/derivative_approximation.cc
deal.II/deal.II/source/numerics/error_estimator.cc
deal.II/deal.II/source/numerics/matrices.cc
deal.II/deal.II/source/numerics/time_dependent.cc
deal.II/doc/development/Makefile
deal.II/doc/development/Makefile.large
deal.II/doc/development/makefiles.1.html
deal.II/doc/doxygen/headers/multithreading.h
deal.II/doc/news/changes-tbb-branch.h [new file with mode: 0644]
deal.II/doc/readme.html
deal.II/examples/step-13/step-13.cc
deal.II/examples/step-14/step-14.cc
deal.II/examples/step-28/step-28.cc
deal.II/examples/step-31/step-31.cc
deal.II/examples/step-32/step-32.cc
deal.II/examples/step-35/Makefile [new file with mode: 0644]
deal.II/examples/step-35/include/EqData.h [new file with mode: 0644]
deal.II/examples/step-35/include/FileReader.h [new file with mode: 0644]
deal.II/examples/step-35/include/MultiComponentFunction.h [new file with mode: 0644]
deal.II/examples/step-35/include/NavierStokes.h [new file with mode: 0644]
deal.II/examples/step-35/parameter-file.prm [new file with mode: 0644]
deal.II/examples/step-35/source/EqData.cc [new file with mode: 0644]
deal.II/examples/step-35/source/EqData_square.ccc [new file with mode: 0644]
deal.II/examples/step-35/source/FileReader.cc [new file with mode: 0644]
deal.II/examples/step-35/source/MultiComponentFunction.cc [new file with mode: 0644]
deal.II/examples/step-35/source/NavierStokes.cc [new file with mode: 0644]
deal.II/examples/step-35/source/main.cc [new file with mode: 0644]
deal.II/examples/step-35/step-35.cc [new file with mode: 0644]
deal.II/examples/step-9/step-9.cc
deal.II/lac/Makefile
deal.II/lac/include/lac/constraint_matrix.h
deal.II/lac/include/lac/sparse_matrix.h
deal.II/lac/include/lac/sparse_matrix.templates.h
deal.II/lac/include/lac/sparse_vanka.templates.h
deal.II/lac/include/lac/swappable_vector.templates.h
deal.II/lac/include/lac/vector.h
deal.II/lac/include/lac/vector.templates.h
deal.II/lac/source/sparse_matrix.cc
deal.II/lac/source/vector.cc
deal.II/lib/Makefile

index de687ebc195dc6ad8da0d7d8051685fc0b1aa13d..4ccc8def6f40eeb4b2fc41c446b2393ed254fe13 100644 (file)
@@ -52,16 +52,16 @@ deps: common/scripts/make_dependencies common/scripts/expand_instantiations
        cd $D/deal.II && $(MAKE) $(MAKEOPTIONS) Makefile.dep
        cd $D/lib && $(MAKE) $(MAKEOPTIONS) external-links
 
-baseg: deps contrib-functionparser
+baseg: deps contrib
        cd $D/base && $(MAKE) $(MAKEOPTIONS) libg
 
-baseo: deps contrib-functionparser
+baseo: deps contrib
        cd $D/base && $(MAKE) $(MAKEOPTIONS) libo
 
-lacg: deps baseg contrib
+lacg: deps baseg
        cd $D/lac && $(MAKE) $(MAKEOPTIONS) libg
 
-laco: deps baseo contrib
+laco: deps baseo
        cd $D/lac && $(MAKE) $(MAKEOPTIONS) libo
 
 1dg: deps baseg lacg
@@ -92,7 +92,7 @@ lac: lacg laco
 $(LIBDIR):
        @mkdir $@
 
-baseg baseo base lacg laco lac 1dg 1do 1d 2dg 2do 2d 3dg 3do 3d all debug contrib -functionparser : $(LIBDIR) deps
+baseg baseo base lacg laco lac 1dg 1do 1d 2dg 2do 2d 3dg 3do 3d all debug contrib functionparser : $(LIBDIR) deps
 
 all: debug optimized
 debug: contrib baseg lacg 2dg 1dg 3dg
@@ -214,8 +214,7 @@ build-test:
        -(./configure $(BUILDTESTFLAGS) 2>&1) > build-test-config
        if test -f common/Make.global_options ; then \
           grep '^enable-shared' common/Make.global_options | grep = ; \
-          grep enable-multithreading common/Make.global_options | grep = ; \
-          grep with-multithreading common/Make.global_options | grep = ; \
+          grep enable-threads common/Make.global_options | grep = ; \
           grep USE_CONTRIB_PETSC common/Make.global_options | grep = ; \
           grep USE_CONTRIB_METIS common/Make.global_options | grep = ; \
           grep USE_CONTRIB_HSL common/Make.global_options | grep = ; \
index c23072956b24c419bc972f3209912996176bad53..dc856965b9306ad1ac2090879f7fffe995b3f0b2 100644 (file)
@@ -644,6 +644,11 @@ AC_DEFUN(DEAL_II_SET_CXX_FLAGS, dnl
       dnl Use -Wno-long-long on Apple Darwin to avoid some unnecessary
       dnl warnings. However, newer gccs on that platform do not have 
       dnl this flag any more, so check whether we can indeed do this
+      dnl
+      dnl Also, the TBB tries to determine whether the system is
+      dnl 64-bit enabled and if so, it builds the object files in 64bit.
+      dnl In order to be able to link with these files, we then have to
+      dnl link with -m64 as well
       *apple-darwin*)
         OLD_CXXFLAGS="$CXXFLAGS"
         CXXFLAGS=-Wno-long-double
@@ -659,6 +664,13 @@ AC_DEFUN(DEAL_II_SET_CXX_FLAGS, dnl
             AC_MSG_RESULT(no)
           ])
 
+        if test "`/usr/sbin/sysctl -n hw.optional.x86_64`" = "1" ; then
+          CXXFLAGS="$CXXFLAGS -m64"
+          CXXFLAGSG="$CXXFLAGSG -m64"
+          CXXFLAGSO="$CXXFLAGSO -m64"
+          LDFLAGS="$LDFLAGS -m64"
+        fi
+
         CXXFLAGS="${OLD_CXXFLAGS}"
         ;;
 
@@ -1516,6 +1528,16 @@ AC_DEFUN(DEAL_II_SET_CC_FLAGS, dnl
         CFLAGSPIC=
         ;;
 
+      *apple-darwin*)
+        dnl The TBB tries to determine whether the system is
+        dnl 64-bit enabled and if so, it builds the object files in 64bit.
+        dnl In order to be able to link with these files, we then have to
+        dnl link with -m64 as well
+        if test "`/usr/sbin/sysctl -n hw.optional.x86_64`" = "1" ; then
+          CFLAGS="$CFLAGS -m64"
+        fi
+        ;;
+
       *)
        CFLAGSPIC="-fPIC"
        ;;
@@ -2082,28 +2104,8 @@ AC_DEFUN(DEAL_II_GET_THREAD_FLAGS, dnl
 
 
 dnl -------------------------------------------------------------
-dnl Test whether multithreading support is requested. This
-dnl does not tell deal.II to actually use it, but the
-dnl compiler flags can be set to allow for it. If the user specified
-dnl --enable-multithreading, then set $enablemultithreading=yes,
-dnl otherwise $enablemultithreading=no.
+dnl Test whether multithreading support is requested.
 dnl
-dnl Usage:
-dnl   DEAL_II_CHECK_MULTITHREADING
-dnl
-dnl -------------------------------------------------------------
-AC_DEFUN(DEAL_II_CHECK_MULTITHREADING, dnl
-[
-  AC_ARG_ENABLE(multithreading,
-  [  --enable-multithreading Set compiler flags to allow for multithreaded
-                          programs],
-    enablemultithreading="$enableval",
-    enablemultithreading=no)
-])
-
-
-
-dnl -------------------------------------------------------------
 dnl If multithreading support is requested, figure out the right
 dnl compiler flags to use:
 dnl - Use the right -threads/-pthread/-mthread option
@@ -2121,12 +2123,38 @@ dnl better to put the flags into the command line, since then we have them
 dnl defined for all include files.
 dnl
 dnl Usage:
-dnl   DEAL_II_SET_MULTITHREADING_FLAGS
+dnl   DEAL_II_CHECK_MULTITHREADING
 dnl
 dnl -------------------------------------------------------------
-AC_DEFUN(DEAL_II_SET_MULTITHREADING_FLAGS, dnl
+AC_DEFUN(DEAL_II_CHECK_MULTITHREADING, dnl
 [
-  if test "$enablemultithreading" = yes ; then
+  AC_ARG_ENABLE(threads,
+    [  --enable-threads  Use multiple threads inside deal.II],
+    [
+      enablethreads="$enableval"
+    ],
+    [
+      dnl Set default to yes unless on cygwin where the TBB is
+      dnl currently not yet ported to
+      case "$target" in
+        *cygwin* )
+          enablethreads=no
+         ;;
+
+       * )
+          enablethreads=yes
+          ;;
+      esac
+    ])
+
+  if test "$enablethreads" = yes ; then
+    dnl On cygwin, the TBB does not compile. Error out.
+    case "$target" in
+      *cygwin* )
+        AC_MSG_ERROR(Multithreading is not supported on CygWin)
+        ;;
+    esac
+
     if test "$GXX" = yes ; then
       DEAL_II_GET_THREAD_FLAGS
       DEAL_II_THREAD_CPPFLAGS
@@ -2166,7 +2194,27 @@ AC_DEFUN(DEAL_II_SET_MULTITHREADING_FLAGS, dnl
            ;;
       esac
     fi
+
+    DEAL_II_CHECK_POSIX_THREAD_FUNCTIONS
+    AC_DEFINE(DEAL_II_USE_MT_POSIX, 1,
+              [Defined if multi-threading is to be achieved by using
+               the POSIX functions])
+
+    dnl In any case, we need to link with libdl since the Threading 
+    dnl Building Blocks require this
+    LDFLAGS="$LDFLAGS -ldl"
   fi
+
+  if test "x$enablethreads" != "xno" ; then
+    DEAL_II_USE_MT_VAL=1
+  else
+    DEAL_II_USE_MT_VAL=0
+  fi
+
+  AC_DEFINE_UNQUOTED(DEAL_II_USE_MT, $DEAL_II_USE_MT_VAL, 
+                     [Flag indicating whether the library shall be
+                      compiled for multithreaded applications. If so,
+                     then it is set to one, otherwise to zero.])
 ])
 
 
@@ -2188,7 +2236,7 @@ dnl
 dnl -------------------------------------------------------------
 AC_DEFUN(DEAL_II_CHECK_PARTLY_BRACKETED_INITIALIZER, dnl
 [
-  if test "$enablemultithreading" = yes ; then
+  if test "$enablethreads" = yes ; then
     case "$GXX_VERSION" in
       gcc*)
        AC_MSG_CHECKING(for only partly bracketed mutex initializer)
@@ -2216,55 +2264,6 @@ AC_DEFUN(DEAL_II_CHECK_PARTLY_BRACKETED_INITIALIZER, dnl
 
 
 
-dnl -------------------------------------------------------------
-dnl Test which library the MT code shall use to support threads.
-dnl We used to support either POSIX or ACE, but support for ACE
-dnl has now been deleted and we only use POSIX these days. However,
-dnl the code to pass another name than "posix" remains here in 
-dnl case someone wants to hook in support for other thread
-dnl implementations
-dnl
-dnl Usage:
-dnl   DEAL_II_CHECK_USE_MT
-dnl
-dnl -------------------------------------------------------------
-AC_DEFUN(DEAL_II_CHECK_USE_MT, dnl
-[
-  AC_ARG_WITH(multithreading,
-  [  --with-multithreading=name  
-                          If name==posix, or no name given, then use POSIX 
-                         threads.],
-      withmultithreading="$withval",
-      withmultithreading=no)
-
-dnl Default (i.e. no arg) means POSIX
-  if test "x$withmultithreading" = "xyes" ; then
-    withmultithreading=posix
-  fi
-
-  if test "x$withmultithreading" != "xno" ; then
-    if test "x$withmultithreading" = "xposix" ; then
-      DEAL_II_CHECK_POSIX_THREAD_FUNCTIONS
-      AC_DEFINE(DEAL_II_USE_MT_POSIX, 1,
-                [Defined if multi-threading is to be achieved by using
-                 the POSIX functions])
-    else
-      AC_MSG_ERROR(Invalid flag for --with-multithreading)
-    fi
-  
-    DEAL_II_USE_MT_VAL=1
-  else
-    DEAL_II_USE_MT_VAL=0
-  fi
-
-  AC_DEFINE_UNQUOTED(DEAL_II_USE_MT, $DEAL_II_USE_MT_VAL, 
-                     [Flag indicating whether the library shall be
-                      compiled for multithreaded applications. If so,
-                     then it is set to one, otherwise to zero.])
-])
-
-
-
 dnl -------------------------------------------------------------
 dnl Check whether the POSIX functions needed for multi-threading
 dnl are available on this system
@@ -4238,6 +4237,48 @@ AC_DEFUN(DEAL_II_CHECK_EXPLICIT_CONSTRUCTOR_BUG, dnl
 
 
 
+dnl -------------------------------------------------------------
+dnl Some older versions of gcc deduce pointers to const functions in
+dnl template contexts to pointer-to-function of const objects.
+dnl This is not correct
+dnl 
+dnl Check for this misfeature.
+dnl 
+dnl Usage: DEAL_II_CHECK_CONST_MEMBER_DEDUCTION_BUG
+dnl 
+dnl --------------------------------------------------------------
+AC_DEFUN(DEAL_II_CHECK_CONST_MEMBER_DEDUCTION_BUG, dnl
+[
+  AC_MSG_CHECKING(for const member deduction bug)
+  AC_LANG(C++)
+  CXXFLAGS="$CXXFLAGSG"
+  AC_TRY_COMPILE(
+    [
+      template <typename T> struct identity { typedef T type; };
+
+      template <typename C> void new_thread (void (C::*fun_ptr)(),
+                                      typename identity<C>::type &c);
+      template <typename C> void new_thread (void (C::*fun_ptr)() const,
+                                      const typename identity<C>::type &c);
+      struct X { void f() const; };
+    ],
+    [
+      X x;
+      new_thread (&X::f, x);
+    ],
+    [
+      AC_MSG_RESULT(no)
+    ],
+    [
+      AC_MSG_RESULT(yes)
+      AC_DEFINE(DEAL_II_CONST_MEMBER_DEDUCTION_BUG, 1, 
+                     [Defined if the compiler has a bug in deducing
+                     the type of pointers to const member functions.])
+    ])
+])
+
+
+
 dnl -------------------------------------------------------------
 dnl Check for GCC bug 36052, see
 dnl   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36052
index 80da5b0661c7e78baaec608844118bd196b121c1..9804c154c7fd7f25b832b31311228a00947af22c 100644 (file)
@@ -17,20 +17,16 @@ h-files      = $(sort $(shell echo include/base/*.h))
 # get options which hold for all files of the project
 include $D/common/Make.global_options
 
+
 # there's an extra .o file for the function parser that we need to link into
-# libbase
+# libbase. do similarly for the threading building block things if threading
+# is enabled
 ifeq ($(enable-parser),yes)
-  extra-o-files = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT)
+  extra-o-files  = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT)
+  extra-g.o-files = $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT)
 endif
 
-
 # production rules
-ifneq ($(with-multithreading),no)
-  MT = MT
-else
-  MT = ==
-endif
-
 $(LIBDIR)/base/%.g.$(OBJEXT) :
        @echo "=====base=============debug======$(MT)== $(<F)"
        @$(CXX) $(CXXFLAGS.g) -c $< -o $@
@@ -59,6 +55,12 @@ ifneq ($(GXX-VERSION),compaq_cxx)
     deplibs.g += $(lib-contrib-trilinos)
     deplibs.o += $(lib-contrib-trilinos)
   endif
+
+  ifeq ($(enable-threads),yes)
+    deplibs.g += $D/lib/libtbb_debug$(shared-lib-suffix)
+    deplibs.o += $D/lib/libtbb$(shared-lib-suffix)
+  endif
+
 else
   deplibs.g =
   deplibs.o =
@@ -77,7 +79,7 @@ $(LIBDIR)/libbase$(static-lib-suffix): $(o-files)
 
 $(LIBDIR)/libbase.g$(static-lib-suffix): $(go-files)
        @echo "=====base=============debug======$(MT)== Linking library:   $(@F)"
-       @$(AR) ru $@ $(go-files) $(extra-o-files)
+       @$(AR) ru $@ $(go-files) $(extra-g.o-files)
        @$(RANLIB) $@
 
 
@@ -88,7 +90,7 @@ $(LIBDIR)/libbase$(shared-lib-suffix): $(o-files)
 
 $(LIBDIR)/libbase.g$(shared-lib-suffix): $(go-files)
        @echo "=====base=============debug======$(MT)== Linking library:   $(@F)"
-       @$(SHLIBLD) $(LDFLAGS) $(SHLIBFLAGS) -o $(LIBDIR)/$(call DEAL_II_SHLIB_NAME,base.g) $(call DEAL_II_ADD_SONAME,base.g) $(go-files) $(extra-o-files)  $(deplibs.g)
+       @$(SHLIBLD) $(LDFLAGS) $(SHLIBFLAGS) -o $(LIBDIR)/$(call DEAL_II_SHLIB_NAME,base.g) $(call DEAL_II_ADD_SONAME,base.g) $(go-files) $(extra-g.o-files)  $(deplibs.g)
        @ln -f -s $(call DEAL_II_SHLIB_NAME,base.g) $@
 
 
index 77b3cfc3f961f9e4991f75f0bf88030869909eb0..b20cc6705333bdc9fac0bc0a11784ff77bcda9e4 100644 (file)
 /* Defined if the compiler supports including <mpi.h> */
 #undef DEAL_II_COMPILER_SUPPORTS_MPI
 
+/* Defined if the compiler has a bug in deducing the type of pointers to const
+   member functions. */
+#undef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+
 /* disable the function parser in contrib */
 #undef DEAL_II_DISABLE_PARSER
 
diff --git a/deal.II/base/include/base/parallel.h b/deal.II/base/include/base/parallel.h
new file mode 100644 (file)
index 0000000..26b6fd7
--- /dev/null
@@ -0,0 +1,917 @@
+//---------------------------------------------------------------------------
+//    $Id: parallel.h 14038 2006-10-23 02:46:34Z bangerth $
+//    Version: $Name$
+//
+//    Copyright (C) 2008, 2009 by the deal.II authors
+//
+//    This file is subject to QPL and may not be  distributed
+//    without copyright and license information. Please refer
+//    to the file deal.II/doc/license.html for the  text  and
+//    further information on this license.
+//
+//---------------------------------------------------------------------------
+#ifndef __deal2__parallel_h
+#define __deal2__parallel_h
+
+
+#include <base/config.h>
+#include <base/exceptions.h>
+#include <base/template_constraints.h>
+
+#include <base/std_cxx1x/tuple.h>
+#include <base/std_cxx1x/bind.h>
+#include <base/std_cxx1x/function.h>
+
+#include <iterator>
+#include <cstddef>
+
+#if DEAL_II_USE_MT == 1
+#  include <tbb/parallel_for.h>
+#  include <tbb/parallel_reduce.h>
+#  include <tbb/partitioner.h>
+#  include <tbb/blocked_range.h>
+#endif
+
+
+//TODO[WB]: allow calling functions to pass along a tbb::affinity_partitioner object to ensure that subsequent calls use the same cache lines
+
+DEAL_II_NAMESPACE_OPEN
+
+/**
+ * A namespace in which we define a few algorithms that can run in parallel
+ * when deal.II is configured to use multiple threads.
+ *
+ * @ingroup threads
+ * @author Wolfgang Bangerth, 2008, 2009
+ */
+namespace parallel
+{
+  namespace internal
+  {
+                                    /**
+                                     * A class that represents a set of
+                                     * iterators each of which are
+                                     * incremented by one at the same
+                                     * time. This is typically used in calls
+                                     * like <code>std::transform(a.begin(),
+                                     * a.end(), b.begin(), functor);</code>
+                                     * where we have synchronous iterators
+                                     * marching through the containers
+                                     * <code>a,b</code>. If an object of this
+                                     * type represents the end of a range,
+                                     * only the first element is considered
+                                     * (we only have <code>a.end()</code>,
+                                     * not <code>b.end()</code>)
+                                     *
+                                     * The template argument of the current
+                                     * class shall be of type
+                                     * <code>std_cxx1x::tuple</code> with
+                                     * arguments equal to the iterator types.
+                                     *
+                                     * This type, and the helper functions
+                                     * associated with it, are used as the
+                                     * Value concept for the blocked_range
+                                     * type of the Threading Building Blocks.
+                                     */
+    template <typename Iterators>
+    struct SynchronousIterators
+    {
+                                        /**
+                                         * Constructor.
+                                         */
+       SynchronousIterators (const Iterators &i);
+
+                                        /**
+                                         * Copy constructor.
+                                         */
+       SynchronousIterators (const SynchronousIterators &i);
+
+                                        /**
+                                         * Storage for the iterators
+                                         * represented by the current class.
+                                         */
+       Iterators iterators;
+    };
+
+
+
+    template <typename Iterators>
+    inline
+    SynchronousIterators<Iterators>::
+    SynchronousIterators (const Iterators &i)
+                   :
+                   iterators (i)
+    {}
+        
+
+    template <typename Iterators>
+    inline
+    SynchronousIterators<Iterators>::
+    SynchronousIterators (const SynchronousIterators &i)
+                   :
+                   iterators (i.iterators)
+    {}
+
+
+
+                                    /**
+                                     * Return whether the first element of
+                                     * the first argument is less than the
+                                     * first element of the second
+                                     * argument. Since the objects compared
+                                     * march forward all elements at the same
+                                     * time, comparing the first element is
+                                     * sufficient.
+                                     */
+    template <typename Iterators>
+    inline
+    bool
+    operator< (const SynchronousIterators<Iterators> &a,
+              const SynchronousIterators<Iterators> &b)
+    {
+      return std_cxx1x::get<0>(a.iterators) < std_cxx1x::get<0>(b.iterators);
+    }
+
+
+
+                                    /**
+                                     * Return the distance between the first
+                                     * and the second argument. Since the
+                                     * objects compared march forward all
+                                     * elements at the same time,
+                                     * differencing the first element is
+                                     * sufficient.
+                                     */
+    template <typename Iterators>
+    inline
+    std::size_t
+    operator- (const SynchronousIterators<Iterators> &a,
+              const SynchronousIterators<Iterators> &b)
+    {
+      Assert (std::distance (std_cxx1x::get<0>(b.iterators),
+                            std_cxx1x::get<0>(a.iterators)) >= 0,
+             ExcInternalError());
+      return std::distance (std_cxx1x::get<0>(b.iterators),
+                           std_cxx1x::get<0>(a.iterators));
+    }
+
+
+                                    /**
+                                     * Advance a tuple of iterators by $n$.
+                                     */
+    template <typename I1, typename I2>
+    inline
+    void advance (std_cxx1x::tuple<I1,I2> &t,
+                 const unsigned int       n)
+    {
+      std::advance (std_cxx1x::get<0>(t), n);
+      std::advance (std_cxx1x::get<1>(t), n);
+    }
+
+                                    /**
+                                     * Advance a tuple of iterators by $n$.
+                                     */
+    template <typename I1, typename I2, typename I3>
+    inline
+    void advance (std_cxx1x::tuple<I1,I2,I3> &t,
+                 const unsigned int          n)
+    {
+      std::advance (std_cxx1x::get<0>(t), n);
+      std::advance (std_cxx1x::get<1>(t), n);
+      std::advance (std_cxx1x::get<2>(t), n);
+    }
+
+                                    /**
+                                     * Advance a tuple of iterators by $n$.
+                                     */
+    template <typename I1, typename I2,
+             typename I3, typename I4>
+    inline
+    void advance (std_cxx1x::tuple<I1,I2,I3, I4> &t,
+                 const unsigned int              n)
+    {
+      std::advance (std_cxx1x::get<0>(t), n);
+      std::advance (std_cxx1x::get<1>(t), n);
+      std::advance (std_cxx1x::get<2>(t), n);
+      std::advance (std_cxx1x::get<3>(t), n);
+    }
+    
+
+
+                                    /**
+                                     * Advance a tuple of iterators by 1.
+                                     */
+    template <typename I1, typename I2>
+    inline
+    void advance_by_one (std_cxx1x::tuple<I1,I2> &t)
+    {
+      ++std_cxx1x::get<0>(t);
+      ++std_cxx1x::get<1>(t);
+    }
+
+                                    /**
+                                     * Advance a tuple of iterators by 1.
+                                     */
+    template <typename I1, typename I2, typename I3>
+    inline
+    void advance_by_one (std_cxx1x::tuple<I1,I2,I3> &t)
+    {
+      ++std_cxx1x::get<0>(t);
+      ++std_cxx1x::get<1>(t);
+      ++std_cxx1x::get<2>(t);
+    }
+    
+                                    /**
+                                     * Advance a tuple of iterators by 1.
+                                     */
+    template <typename I1, typename I2,
+             typename I3, typename I4>
+    inline
+    void advance_by_one (std_cxx1x::tuple<I1,I2,I3,I4> &t)
+    {
+      ++std_cxx1x::get<0>(t);
+      ++std_cxx1x::get<1>(t);
+      ++std_cxx1x::get<2>(t);
+      ++std_cxx1x::get<3>(t);
+    }
+    
+
+
+                                    /**
+                                     * Advance the elements of this iterator
+                                     * by $n$.
+                                     */
+    template <typename Iterators>
+    inline
+    SynchronousIterators<Iterators>
+    operator + (const SynchronousIterators<Iterators> &a,
+               const std::size_t                      n)
+    {
+      SynchronousIterators<Iterators> x (a);
+      parallel::internal::advance (x.iterators, n);
+      return x;
+    }
+
+                                    /**
+                                     * Advance the elements of this iterator
+                                     * by 1.
+                                     */
+    template <typename Iterators>
+    inline
+    SynchronousIterators<Iterators>
+    operator ++ (SynchronousIterators<Iterators> &a)
+    {
+      parallel::internal::advance_by_one (a.iterators);
+      return a;
+    }
+
+
+                                    /**
+                                     * Compare synch iterators for
+                                     * inequality. Since they march in synch,
+                                     * comparing only the first element is
+                                     * sufficient.
+                                     */
+    template <typename Iterators>
+    inline
+    bool
+    operator != (const SynchronousIterators<Iterators> &a,
+                const SynchronousIterators<Iterators> &b)
+    {
+      return (std_cxx1x::get<0>(a.iterators) !=
+             std_cxx1x::get<0>(b.iterators));
+    }
+    
+
+                                    /**
+                                     * Convert a function object of type F
+                                     * into an object that can be applied to
+                                     * all elements of a range of synchronous
+                                     * iterators.
+                                     */
+    template <typename F>
+    struct Body
+    {
+                                        /**
+                                         * Constructor. Take and package the
+                                         * given function object.
+                                         */
+       Body (const F &f)
+                       :
+                       f (f)
+         {}
+
+       template <typename Range>
+       void
+       operator () (const Range &range) const
+         {
+           for (typename Range::const_iterator p=range.begin();
+                p != range.end(); ++p)
+             apply (f, p.iterators);
+         }
+       
+      private:
+                                        /**
+                                         * The stored function object.
+                                         */
+       const F f;
+
+                                        /**
+                                         * Apply F to a set of iterators with
+                                         * two elements.
+                                         */
+       template <typename I1, typename I2>
+       static
+       void
+       apply (const F &f,
+              const std_cxx1x::tuple<I1,I2> &p)
+         {
+           *std_cxx1x::get<1>(p) = f (*std_cxx1x::get<0>(p));
+         }
+
+                                        /**
+                                         * Apply F to a set of iterators with
+                                         * three elements.
+                                         */
+       template <typename I1, typename I2, typename I3>
+       static
+       void
+       apply (const F &f,
+              const std_cxx1x::tuple<I1,I2,I3> &p)
+         {
+           *std_cxx1x::get<2>(p) = f (*std_cxx1x::get<0>(p),
+                                      *std_cxx1x::get<1>(p));
+         }
+
+                                        /**
+                                         * Apply F to a set of iterators with
+                                         * three elements.
+                                         */
+       template <typename I1, typename I2,
+                 typename I3, typename I4>
+       static
+       void
+       apply (const F &f,
+              const std_cxx1x::tuple<I1,I2,I3,I4> &p)
+         {
+           *std_cxx1x::get<3>(p) = f (*std_cxx1x::get<0>(p),
+                                      *std_cxx1x::get<1>(p),
+                                      *std_cxx1x::get<2>(p));
+         }
+    };
+
+
+                                    /**
+                                     * Take a function object and create a
+                                     * Body object from it. We do this in
+                                     * this helper function since
+                                     * alternatively we would have to specify
+                                     * the actual data type of F -- which for
+                                     * function objects is often
+                                     * extraordinarily complicated.
+                                     */
+    template <typename F>
+    Body<F> make_body(const F &f)
+    {
+      return Body<F>(f);
+    }
+  }
+  
+                                  /**
+                                   * An algorithm that performs the action
+                                   * <code>*out++ = predicate(*in++)</code>
+                                   * where the <code>in</code> iterator
+                                   * ranges over the given input range.
+                                   *
+                                   * This algorithm does pretty much what
+                                   * std::transform does. The difference is
+                                   * that the function can run in parallel
+                                   * when deal.II is configured to use
+                                   * multiple threads. In that case, the last
+                                   * argument denotes the minimum number each
+                                   * thread can work on; the number must be
+                                   * large enough to amortize the startup
+                                   * cost of new threads, and small enough to
+                                   * ensure that (i) threads will be started
+                                   * at all, and (ii) threads can be
+                                   * reasonably load balanced.
+                                   *
+                                   * For a discussion of the kind of
+                                   * problems to which this function
+                                   * is applicable, see the
+                                   * @ref threads "Parallel computing with multiple processors"
+                                   * module.
+                                   */
+  template <typename InputIterator,
+           typename OutputIterator,
+           typename Predicate>
+  void transform (const InputIterator &begin_in,
+                 const InputIterator &end_in,
+                 OutputIterator       out,
+                 Predicate           &predicate,
+                 const unsigned int   grainsize)
+  {
+#if DEAL_II_USE_MT == 0
+                                    // make sure we don't get compiler
+                                    // warnings about unused arguments
+    (void) grainsize;
+
+    for (OutputIterator in = begin_in; in != end_in;)
+      *out++ = predicate (*in++);
+#else
+    typedef std_cxx1x::tuple<InputIterator,OutputIterator> Iterators;
+    typedef internal::SynchronousIterators<Iterators> SyncIterators;
+    Iterators x_begin (begin_in, out);
+    Iterators x_end (end_in, OutputIterator());
+    tbb::parallel_for (tbb::blocked_range<SyncIterators>(x_begin,
+                                                        x_end,
+                                                        grainsize),
+                      internal::make_body (predicate),
+                      tbb::auto_partitioner());
+#endif
+  }
+
+
+
+                                  /**
+                                   * An algorithm that performs the action
+                                   * <code>*out++ = predicate(*in1++, *in2++)</code>
+                                   * where the <code>in1</code> iterator
+                                   * ranges over the given input range.
+                                   *
+                                   * This algorithm does pretty much what
+                                   * std::transform does. The difference is
+                                   * that the function can run in parallel
+                                   * when deal.II is configured to use
+                                   * multiple threads. In that case, the last
+                                   * argument denotes the minimum number each
+                                   * thread can work on; the number must be
+                                   * large enough to amortize the startup
+                                   * cost of new threads, and small enough to
+                                   * ensure that (i) threads will be started
+                                   * at all, and (ii) threads can be
+                                   * reasonably load balanced.
+                                   *
+                                   * For a discussion of the kind of
+                                   * problems to which this function
+                                   * is applicable, see the
+                                   * @ref threads "Parallel computing with multiple processors"
+                                   * module.
+                                   */
+  template <typename InputIterator1,
+           typename InputIterator2,
+           typename OutputIterator,
+           typename Predicate>
+  void transform (const InputIterator1 &begin_in1,
+                 const InputIterator1 &end_in1,
+                 InputIterator2        in2,
+                 OutputIterator        out,
+                 Predicate            &predicate,
+                 const unsigned int    grainsize)
+  {
+#if DEAL_II_USE_MT == 0
+                                    // make sure we don't get compiler
+                                    // warnings about unused arguments
+    (void) grainsize;
+
+    for (OutputIterator in1 = begin_in1; in1 != end_in1;)
+      *out++ = predicate (*in1++, *in2++);
+#else
+    typedef
+      std_cxx1x::tuple<InputIterator1,InputIterator2,OutputIterator>
+      Iterators;
+    typedef internal::SynchronousIterators<Iterators> SyncIterators;
+    Iterators x_begin (begin_in1, in2, out);
+    Iterators x_end (end_in1, InputIterator2(), OutputIterator());
+    tbb::parallel_for (tbb::blocked_range<SyncIterators>(x_begin,
+                                                        x_end,
+                                                        grainsize),
+                      internal::make_body (predicate),
+                      tbb::auto_partitioner());
+#endif
+  }
+
+
+
+                                  /**
+                                   * An algorithm that performs the action
+                                   * <code>*out++ = predicate(*in1++, *in2++, *in3++)</code>
+                                   * where the <code>in1</code> iterator
+                                   * ranges over the given input range.
+                                   *
+                                   * This algorithm does pretty much what
+                                   * std::transform does. The difference is
+                                   * that the function can run in parallel
+                                   * when deal.II is configured to use
+                                   * multiple threads. In that case, the last
+                                   * argument denotes the minimum number each
+                                   * thread can work on; the number must be
+                                   * large enough to amortize the startup
+                                   * cost of new threads, and small enough to
+                                   * ensure that (i) threads will be started
+                                   * at all, and (ii) threads can be
+                                   * reasonably load balanced.
+                                   *
+                                   * For a discussion of the kind of
+                                   * problems to which this function
+                                   * is applicable, see the
+                                   * @ref threads "Parallel computing with multiple processors"
+                                   * module.
+                                   */
+  template <typename InputIterator1,
+           typename InputIterator2,
+           typename InputIterator3,
+           typename OutputIterator,
+           typename Predicate>
+  void transform (const InputIterator1 &begin_in1,
+                 const InputIterator1 &end_in1,
+                 InputIterator2        in2,
+                 InputIterator3        in3,
+                 OutputIterator        out,
+                 Predicate            &predicate,
+                 const unsigned int    grainsize)
+  {
+#if DEAL_II_USE_MT == 0
+                                    // make sure we don't get compiler
+                                    // warnings about unused arguments
+    (void) grainsize;
+
+    for (OutputIterator in1 = begin_in1; in1 != end_in1;)
+      *out++ = predicate (*in1++, *in2++, *in3++);
+#else
+    typedef
+      std_cxx1x::tuple<InputIterator1,InputIterator2,InputIterator3,OutputIterator>
+      Iterators;
+    typedef internal::SynchronousIterators<Iterators> SyncIterators;
+    Iterators x_begin (begin_in1, in2, in3, out);
+    Iterators x_end (end_in1, InputIterator2(),
+                    InputIterator3(), OutputIterator());
+    tbb::parallel_for (tbb::blocked_range<SyncIterators>(x_begin,
+                                                        x_end,
+                                                        grainsize),
+                      internal::make_body (predicate),
+                      tbb::auto_partitioner());
+#endif
+  }
+
+
+  namespace internal
+  {
+#if DEAL_II_USE_MT
+                                    /**
+                                     * Take a range argument and call the
+                                     * given function with its begin and end.
+                                     */
+    template <typename RangeType, typename Function>
+    void apply_to_subranges (const tbb::blocked_range<RangeType> &range,
+                            const Function  &f)
+    {
+      f (range.begin(), range.end());
+    }
+#endif
+  }
+
+
+                                  /**
+                                   * This function applies the given function
+                                   * argument @p f to all elements in the range
+                                   * <code>[begin,end)</code> and may do so
+                                   * in parallel.
+                                   *
+                                   * However, in many cases it is not
+                                   * efficient to call a function on each
+                                   * element, so this function calls the
+                                   * given function object on sub-ranges. In
+                                   * other words: if the given range
+                                   * <code>[begin,end)</code> is smaller than
+                                   * grainsize or if multithreading is not
+                                   * enabled, then we call
+                                   * <code>f(begin,end)</code>; otherwise, we
+                                   * may execute, possibly in %parallel, a
+                                   * sequence of calls <code>f(b,e)</code>
+                                   * where <code>[b,e)</code> are
+                                   * subintervals of <code>[begin,end)</code>
+                                   * and the collection of calls we do to
+                                   * <code>f(.,.)</code> will happen on
+                                   * disjoint subintervals that collectively
+                                   * cover the original interval
+                                   * <code>[begin,end)</code>.
+                                   *
+                                   * Oftentimes, the called function will of
+                                   * course have to get additional
+                                   * information, such as the object to work
+                                   * on for a given value of the iterator
+                                   * argument. This can be achieved by
+                                   * <i>binding</i> certain arguments. For
+                                   * example, here is an implementation of a
+                                   * matrix-vector multiplication $y=Ax$ for
+                                   * a full matrix $A$ and vectors $x,y$:
+                                   * @code
+                                   *   void matrix_vector_product (const FullMatrix &A,
+                                   *                               const Vector     &x,
+                                   *                               Vector           &y)
+                                   *   {
+                                   *     parallel::apply_to_subranges
+                                   *        (0, A.n_rows(),
+                                   *         std_cxx1x::bind (&mat_vec_on_subranges,
+                                   *                          _1, _2,
+                                   *                          std_cxx1x::cref(A),
+                                   *                          std_cxx1x::cref(x),
+                                   *                          std_cxx1x::ref(y)),
+                                   *         50);
+                                   *   }
+                                   *
+                                   *   void mat_vec_on_subranges (const unsigned int begin_row,
+                                   *                              const unsigned int end_row,
+                                   *                              const FullMatrix &A,
+                                   *                              const Vector     &x,
+                                   *                              Vector           &y)
+                                   *   {
+                                   *     for (unsigned int row=begin_row; row!=end_row; ++row)
+                                   *       for (unsigned int col=0; col<x.size(); ++col)
+                                   *         y(row) += A(row,col) * x(col);
+                                   *   }
+                                   * @endcode
+                                   *
+                                   * Note how we use the
+                                   * <code>std_cxx1x::bind</code> function to
+                                   * convert
+                                   * <code>mat_vec_on_subranged</code> from a
+                                   * function that takes 5 arguments to one
+                                   * taking 2 by binding the remaining
+                                   * arguments (the modifiers
+                                   * <code>std_cxx1x::ref</code> and
+                                   * <code>std_cxx1x::cref</code> make sure
+                                   * that the enclosed variables are actually
+                                   * passed by reference and constant
+                                   * reference, rather than by value). The
+                                   * resulting function object requires only
+                                   * two arguments, begin_row and end_row,
+                                   * with all other arguments fixed.
+                                   *
+                                   * The code, if in single-thread mode, will
+                                   * call <code>mat_vec_on_subranges</code>
+                                   * on the entire range
+                                   * <code>[0,n_rows)</code> exactly once. In
+                                   * multi-threaded mode, however, it may be
+                                   * called multiple times on subranges of
+                                   * this interval, possibly allowing more
+                                   * than one CPU core to take care of part
+                                   * of the work.
+                                   *
+                                   * The @p grainsize argument (50 in the
+                                   * example above) makes sure that subranges
+                                   * do not become too small, to avoid
+                                   * spending more time on scheduling
+                                   * subranges to CPU resources than on doing
+                                   * actual work.
+                                   *
+                                   * For a discussion of the kind of
+                                   * problems to which this function
+                                   * is applicable, see also the
+                                   * @ref threads "Parallel computing with multiple processors"
+                                   * module.
+                                   */
+  template <typename RangeType, typename Function>
+  void apply_to_subranges (const RangeType                          &begin,
+                          const typename identity<RangeType>::type &end,
+                          const Function                           &f,
+                          const unsigned int                        grainsize)
+  {
+#if DEAL_II_USE_MT == 0
+                                    // make sure we don't get compiler
+                                    // warnings about unused arguments
+    (void) grainsize;
+
+    f (begin, end);
+#else
+    tbb::parallel_for (tbb::blocked_range<RangeType>
+                      (begin, end, grainsize),
+                      std_cxx1x::bind (&internal::apply_to_subranges<RangeType,Function>,
+                                       _1,
+                                       std_cxx1x::cref(f)),
+                      tbb::auto_partitioner());
+#endif
+  }
+
+
+  
+  namespace internal
+  {  
+#if DEAL_II_USE_MT == 1
+                                    /**
+                                     * A class that conforms to the Body
+                                     * requirements of the TBB
+                                     * parallel_reduce function. The first
+                                     * template argument denotes the type on
+                                     * which the reduction is to be done. The
+                                     * second denotes the type of the
+                                     * function object that shall be called
+                                     * for each subrange.
+                                     */
+    template <typename ResultType,
+             typename Function>
+    struct ReductionOnSubranges
+    {
+                                        /**
+                                         * A variable that will hold the
+                                         * result of the reduction.
+                                         */
+       ResultType result;
+
+                                        /**
+                                         * Constructor. Take the function
+                                         * object to call on each sub-range
+                                         * as well as the neutral element
+                                         * with respect to the reduction
+                                         * operation.
+                                         *
+                                         * The second argument denotes a
+                                         * function object that will be use
+                                         * to reduce the result of two
+                                         * computations into one number. An
+                                         * example if we want to simply
+                                         * accumulate integer results would
+                                         * be std::plus<int>().
+                                         */
+       template <typename Reductor>
+       ReductionOnSubranges (const Function &f,
+                             const Reductor &reductor,
+                             const ResultType neutral_element = ResultType())
+                       :
+                       result (neutral_element),
+                       f (f),
+                       neutral_element (neutral_element),
+                       reductor (reductor)
+         {}
+
+                                        /**
+                                         * Splitting constructor. See the TBB
+                                         * book for more details about this.
+                                         */
+       ReductionOnSubranges (const ReductionOnSubranges &r,
+                             tbb::split)
+                       :
+                       result (r.neutral_element),
+                       f (r.f),
+                       neutral_element (r.neutral_element),
+                       reductor (r.reductor)
+         {}
+       
+                                        /**
+                                         * Join operation: merge the results
+                                         * from computations on different
+                                         * sub-intervals.
+                                         */
+       void join (const ReductionOnSubranges &r)
+         {
+           result = reductor(result, r.result);
+         }
+       
+                                        /**
+                                         * Execute the given function on the
+                                         * specified range.
+                                         */
+       template <typename RangeType>
+       void operator () (const tbb::blocked_range<RangeType> &range)
+         {
+           result = reductor(result,
+                             f (range.begin(), range.end()));
+         }
+
+      private:
+                                        /**
+                                         * The function object to call on
+                                         * every sub-range.
+                                         */
+       const Function f;
+       
+                                        /**
+                                         * The neutral element with respect
+                                         * to the reduction operation. This
+                                         * is needed when calling the
+                                         * splitting constructor since we
+                                         * have to re-set the result variable
+                                         * in this case.
+                                         */
+       const ResultType neutral_element;
+
+                                        /**
+                                         * The function object to be used to
+                                         * reduce the result of two calls
+                                         * into one number.
+                                         */
+       const std_cxx1x::function<ResultType (ResultType, ResultType)> reductor;
+    };
+#endif  
+  }
+  
+
+                                  /**
+                                   * This function works a lot like the
+                                   * apply_to_subranges(), but it allows to
+                                   * accumulate numerical results computed on
+                                   * each subrange into one number. The type
+                                   * of this number is given by the
+                                   * ResultType template argument that needs
+                                   * to be explicitly specified.
+                                   *
+                                   * An example of use of this function is to
+                                   * compute the value of the expression $x^T
+                                   * A x$ for a square matrix $A$ and a
+                                   * vector $x$. The sum over rows can be
+                                   * parallelized and the whole code might
+                                   * look like this:
+                                   * @code
+                                   *   void matrix_norm (const FullMatrix &A,
+                                   *                     const Vector     &x)
+                                   *   {
+                                   *     return
+                                   *      std::sqrt
+                                   *       (parallel::accumulate_from_subranges<double>
+                                   *        (0, A.n_rows(),
+                                   *         std_cxx1x::bind (&mat_norm_sqr_on_subranges,
+                                   *                          _1, _2,
+                                   *                          std_cxx1x::cref(A),
+                                   *                          std_cxx1x::cref(x)),
+                                   *         50);
+                                   *   }
+                                   *
+                                   *   double
+                                   *   mat_norm_sqr_on_subranges (const unsigned int begin_row,
+                                   *                              const unsigned int end_row,
+                                   *                              const FullMatrix &A,
+                                   *                              const Vector     &x)
+                                   *   {
+                                   *     double norm_sqr = 0;
+                                   *     for (unsigned int row=begin_row; row!=end_row; ++row)
+                                   *       for (unsigned int col=0; col<x.size(); ++col)
+                                   *         norm_sqr += x(row) * A(row,col) * x(col);
+                                   *     return norm_sqr;
+                                   *   }
+                                   * @endcode
+                                   *
+                                   * Here,
+                                   * <code>mat_norm_sqr_on_subranges</code>
+                                   * is called on the entire range
+                                   * <code>[0,A.n_rows())</code> if this
+                                   * range is less than the minimum grainsize
+                                   * (above chosen as 50) or if deal.II is
+                                   * configured to not use
+                                   * multithreading. Otherwise, it may be
+                                   * called on subsets of the given range,
+                                   * with results from the individual
+                                   * subranges accumulated internally.
+                                   *
+                                   * @warning If ResultType is a floating point
+                                   * type, then accumulation is not a
+                                   * commutative operation. In other words,
+                                   * if the given function object is called
+                                   * three times on three subranges,
+                                   * returning values $a,b,c$, then the
+                                   * returned result of this function is
+                                   * $a+b+c$. However, depending on how the
+                                   * three sub-tasks are distributed on
+                                   * available CPU resources, the result may
+                                   * also be $a+c+b$ or any other
+                                   * permutation; because floating point
+                                   * addition does not commute (as oppose, of
+                                   * course, to addition of real %numbers),
+                                   * the result of invoking this function
+                                   * several times may differ on the order of
+                                   * round-off.
+                                   *
+                                   * For a discussion of the kind of
+                                   * problems to which this function
+                                   * is applicable, see also the
+                                   * @ref threads "Parallel computing with multiple processors"
+                                   * module.
+                                   */
+  template <typename ResultType, typename RangeType, typename Function>
+  ResultType accumulate_from_subranges (const Function &f,
+                                       const RangeType                          &begin,
+                                       const typename identity<RangeType>::type &end,
+                                       const unsigned int grainsize)
+  {
+#if DEAL_II_USE_MT == 0
+                                    // make sure we don't get compiler
+                                    // warnings about unused arguments
+    (void) grainsize;
+
+    return f(begin,end);
+#else
+    internal::ReductionOnSubranges<ResultType,Function>
+      reductor (f, std::plus<ResultType>(), 0);
+    tbb::parallel_reduce (tbb::blocked_range<RangeType>(begin, end, grainsize),
+                         reductor,
+                         tbb::auto_partitioner());
+    return reductor.result;
+#endif
+  }
+  
+}
+
+
+DEAL_II_NAMESPACE_CLOSE
+
+#endif
index 9c040a0f690611451c0fb75c60e0deef9a55081a..223a8a959057ec5c6de9a56c4bc238e0c3aed112 100644 (file)
@@ -101,6 +101,65 @@ template <typename T> struct constraint_and_return_value<false,T>
 
 
 
+/**
+ * A template class that simply exports its template argument as a local
+ * typedef. This class, while at first appearing useless, makes sense in the
+ * following context: if you have a function template as follows:
+ * @code
+ *   template <typename T> void f(T, T);
+ * @endcode
+ * then it can't be called in an expression like <code>f(1, 3.141)</code>
+ * because the type <code>T</code> of the template can not be deduced
+ * in a unique way from the types of the arguments. However, if the
+ * template is written as
+ * @code
+ *   template <typename T> void f(T, typename identity<T>::type);
+ * @endcode
+ * then the call becomes valid: the type <code>T</code> is not deducible
+ * from the second argument to the function, so only the first argument
+ * participates in template type resolution.
+ *
+ * The context for this feature is as follows: consider
+ * @code
+ * template <typename RT, typename A>
+ * void forward_call(RT (*p) (A), A a)  { p(a); }
+ *
+ * void h (double);
+ *
+ * void g()
+ * {
+ *   forward_call(&h, 1);
+ * }
+ * @endcode
+ * This code fails to compile because the compiler can't decide whether the
+ * template type <code>A</code> should be <code>double</code> (from the
+ * signature of the function given as first argument to
+ * <code>forward_call</code>, or <code>int</code> because the expression
+ * <code>1</code> has that type. Of course, what we would like the compiler
+ * to do is simply cast the <code>1</code> to <code>double</code>. We can
+ * achieve this by writing the code as follows:
+ * @code
+ * template <typename RT, typename A>
+ * void forward_call(RT (*p) (A), typename identity<A>::type a)  { p(a); }
+ *
+ * void h (double);
+ *
+ * void g()
+ * {
+ *   forward_call(&h, 1);
+ * }
+ * @endcode
+ *
+ * @author Wolfgang Bangerth, 2008
+ */
+template <typename T>
+struct identity
+{
+    typedef T type;
+};
+
+
+
 /**
  * A class to perform comparisons of arbitrary pointers for equality. In some
  * circumstances, one would like to make sure that two arguments to a function
index 99ae5fab6870a1b38736464f9ac6ad8e94bd8270..d60a00de41469d0371d300b0291064c03a64c6a7 100644 (file)
@@ -16,6 +16,7 @@
 
 #include <base/config.h>
 #include <base/exceptions.h>
+#include <base/template_constraints.h>
 #include <base/std_cxx1x/tuple.h>
 #include <base/std_cxx1x/function.h>
 #include <base/std_cxx1x/shared_ptr.h>
 #include <utility>
 
 
+#if DEAL_II_USE_MT == 1
+#  if defined(DEAL_II_USE_MT_POSIX)
+#    include <pthread.h>
+#  endif
+#  include <tbb/task.h>
+#  include <tbb/task_scheduler_init.h>
+#endif
+
+
+
 DEAL_II_NAMESPACE_OPEN
 
+/*!@addtogroup threads */
+/*@{*/
+
 
 /**
  * A namespace for the implementation of thread management in
@@ -48,14 +62,14 @@ namespace Threads
 {
 /**
  * This class is used instead of a true lock class when not using
- * multithreading. It allows to write programs such that they start
- * new threads and/or lock objects in multithreading mode, and use
- * dummy thread management and synchronisation classes instead when
- * running in single-thread mode. Specifically, the <tt>spawn</tt> functions
- * only call the function but wait for it to return instead of running
- * in on another thread, and the mutices do nothing really. The only
- * reason to provide such a function is that the program can be
- * compiled both in MT and non-MT mode without difference.
+ * multithreading. It allows to write programs such that they start new
+ * threads and/or lock objects in multithreading mode, and use dummy thread
+ * management and synchronisation classes instead when running in
+ * single-thread mode. Specifically, the <tt>new_thread</tt> functions only
+ * call the function but wait for it to return instead of running in on
+ * another thread, and the mutices do nothing really. The only reason to
+ * provide such a function is that the program can be compiled both in MT and
+ * non-MT mode without difference.
  *
  * @author Wolfgang Bangerth, 2000, 2003
  */
@@ -142,7 +156,7 @@ namespace Threads
  * programs such that they start new threads and/or lock objects in
  * multithreading mode, and use dummy thread management and
  * synchronisation classes instead when running in single-thread
- * mode. Specifically, the <tt>spawn</tt> functions only call the function
+ * mode. Specifically, the <tt>new_thread</tt> functions only call the function
  * but wait for it to return instead of running in on another thread,
  * and the mutices do nothing really. The only reason to provide such
  * a function is that the program can be compiled both in MT and
@@ -626,9 +640,8 @@ namespace Threads
                                     * Upon program start, this number
                                     * is one. It is increased each
                                     * time a thread is created using
-                                    * the Threads::spawn or
-                                    * Threads::spawn_n()
-                                    * functions. It is decreased once
+                                    * the Threads::new_thread
+                                    * function. It is decreased once
                                     * a thread terminates by returning
                                     * from the function that was
                                     * spawned.
@@ -649,6 +662,8 @@ namespace Threads
                                     * thread), then these events are
                                     * not registered and counted for
                                     * the result of this function.
+                                   *
+                                   * @ingroup threads
                                     */
   unsigned int n_existing_threads ();
 
@@ -666,6 +681,8 @@ namespace Threads
                                    * systems seems to support
                                    * <tt>gettid</tt>, so that part of
                                    * the code is untested yet.
+                                   *
+                                   * @ingroup threads
                                    */
   unsigned int this_thread_id ();
   
@@ -689,6 +706,8 @@ namespace Threads
                                    * iterators, where each pair
                                    * denotes the range
                                    * <tt>[begin[i],end[i])</tt>.
+                                   *
+                                   * @ingroup threads
                                    */
   template <typename ForwardIterator>
   std::vector<std::pair<ForwardIterator,ForwardIterator> >
@@ -704,6 +723,8 @@ namespace Threads
                                    * the difference that instead of
                                    * iterators, now values are taken
                                    * that define the whole interval.
+                                   *
+                                   * @ingroup threads                              
                                    */
   std::vector<std::pair<unsigned int,unsigned int> >
   split_interval (const unsigned int begin,
@@ -1326,6 +1347,34 @@ namespace Threads
        void thread_entry_point (const std_cxx1x::function<RT ()> function,
                                 ThreadDescriptor<RT> *descriptor)
          {
+                                            // create a new scheduler object
+                                            // so that we can start tasks on
+                                            // the new thread. the scheduler
+                                            // will go out of scope at the
+                                            // end of the function, which
+                                            // also coincides with the end of
+                                            // the thread
+                                            //
+                                            // one may think that the
+                                            // creation of a scheduler is
+                                            // expensive, but just creating
+                                            // one and then destroying it
+                                            // again costs about 92
+                                            // nanoseconds on my laptop
+                                            // (early 2009), whereas thread
+                                            // creation takes about 12000
+                                            // nanoseconds, more than 100
+                                            // times longer. there is
+                                            // therefore no need to only
+                                            // create the scheduler when this
+                                            // is actually necessary because
+                                            // we are spawning tasks on this
+                                            // particular thread; rather, we
+                                            // make our life simpler by
+                                            // always having a scheduler
+                                            // around
+           tbb::task_scheduler_init scheduler;
+
                                             // now call the function
                                             // in question. since an
                                              // exception that is
@@ -1452,6 +1501,7 @@ namespace Threads
                                     * 
                                     * @author Wolfgang Bangerth, 2003, 2009
                                    * @ingroup threads
+                                   * @ingroup threads
                                     */
   template <typename RT = void>
   class Thread
@@ -2045,6 +2095,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with no arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT>
   inline
@@ -2059,6 +2111,8 @@ namespace Threads
                                     * Overload of the non-const spawn
                                     * function for member functions with
                                     * no arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT, typename C>
   inline
@@ -2074,6 +2128,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * const member functions with no
                                     * arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT, typename C>
   inline
@@ -2094,6 +2150,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 1 argument.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT, typename Arg1>
   inline
@@ -2109,6 +2167,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 1 argument.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C, typename Arg1>
   inline
@@ -2124,6 +2184,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 1
                                    * argument.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C, typename Arg1>
   inline
@@ -2142,6 +2204,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 2 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT, typename Arg1, typename Arg2>
   inline
@@ -2157,6 +2221,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 2 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C, typename Arg1, typename Arg2>
   inline
@@ -2172,6 +2238,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 2
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C, typename Arg1, typename Arg2>
   inline
@@ -2190,6 +2258,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 3 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3>
@@ -2206,6 +2276,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 3 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3>
@@ -2222,6 +2294,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 3
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3>
@@ -2242,6 +2316,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 4 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3, typename Arg4>
@@ -2258,6 +2334,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 4 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3, typename Arg4>
@@ -2274,6 +2352,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 4
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3, typename Arg4>
@@ -2293,6 +2373,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 5 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3,
@@ -2310,6 +2392,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 5 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2327,6 +2411,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 5
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2347,6 +2433,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 6 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3,
@@ -2364,6 +2452,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 6 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2381,6 +2471,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 6
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2402,6 +2494,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 7 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3,
@@ -2421,6 +2515,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 7 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2440,6 +2536,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 7
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2463,6 +2561,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 8 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3,
@@ -2484,6 +2584,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 8 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2505,6 +2607,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 8
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2530,6 +2634,8 @@ namespace Threads
                                     * Overload of the spawn function for
                                     * non-member or static member
                                     * functions with 9 arguments.
+                                   *
+                                   * @deprecated
                                     */
   template <typename RT,
             typename Arg1, typename Arg2, typename Arg3,
@@ -2551,6 +2657,8 @@ namespace Threads
                                    * Overload of the non-const spawn
                                    * function for member functions with
                                    * 9 arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2572,6 +2680,8 @@ namespace Threads
                                    * Overload of the spawn function for
                                    * const member functions with 9
                                    * arguments.
+                                   *
+                                   * @deprecated
                                    */
   template <typename RT, typename C,
            typename Arg1, typename Arg2, typename Arg3,
@@ -2589,61 +2699,2453 @@ namespace Threads
       std_cxx1x::function<typename internal::fun_ptr<RT,std_cxx1x::tuple<Arg1, Arg2, Arg3, Arg4, Arg5, Arg6, Arg7, Arg8, Arg9> >::type>
       (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c), _1, _2, _3, _4, _5, _6, _7, _8, _9));
   }
+
+
+
+// ----------- thread starters for functions not taking any parameters
+
+                                  /**
+                                    * Overload of the new_thread function for
+                                    * objects that can be converted to
+                                    * std_cxx1x::function<RT ()>, i.e. anything
+                                    * that can be called like a function
+                                    * object without arguments and returning
+                                    * an object of type RT (or void).
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT>
+  inline
+  Thread<RT>
+  new_thread (const std_cxx1x::function<RT ()> &function)
+  {
+    return Thread<RT>(function);
+  }
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with no arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)())
+  {
+    return Thread<RT>(fun_ptr);
+  }
   
 
+                                   /**
+                                    * Overload of the non-const new_thread
+                                    * function for member functions with
+                                    * no arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(),
+             typename identity<C>::type &c)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c)));
+  }
+
 
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
                                    /**
-                                    * A container for thread
-                                    * objects. Allows to add new
-                                    * thread objects and wait for them
-                                    * all together. The thread objects
-                                    * need to have the same return
-                                    * value for the called function.
+                                    * Overload of the new_thread function for
+                                    * const member functions with no
+                                    * arguments.
                                     *
-                                    * @author Wolfgang Bangerth, 2003
-                                    */
-  template <typename RT = void>
-  class ThreadGroup 
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)() const,
+             const typename identity<C>::type &c)
   {
-    public:
-                                       /**
-                                        * Add another thread object to
-                                        * the collection.
-                                        */
-      ThreadGroup & operator += (const Thread<RT> &t)
-       {
-         threads.push_back (t);
-         return *this;
-       }
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c)));
+  }
+#endif  
+
+
+
+// ----------- thread starters for unary functions
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 1 argument.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename Arg1>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1),
+             typename identity<Arg1>::type arg1)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 1 argument.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 1
+                                   * argument.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+#endif
+
+// ----------- thread starters for binary functions
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 2 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename Arg1, typename Arg2>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2)));
+  }
+
 
-                                       /**
-                                        * Wait for all threads in the
-                                        * collection to finish. It is
-                                        * not a problem if some of
-                                        * them have already been
-                                        * waited for, i.e. you may
-                                        * call this function more than
-                                        * once, and you can also add
-                                        * new thread objects between
-                                        * subsequent calls to this
-                                        * function if you want.
-                                        */
-      void join_all () const {
-        for (typename std::list<Thread<RT> >::const_iterator
-               t=threads.begin(); t!=threads.end(); ++t)
-          t->join ();
-      }
-    
-    private:
-                                       /**
-                                        * List of thread objects.
-                                        */
-      std::list<Thread<RT> > threads;
-  };
-  
-  
-}   // end of implementation of namespace Threads
 
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 2 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1, typename Arg2>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 2
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1, typename Arg2>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2)));
+  }  
+#endif
+
+// ----------- thread starters for ternary functions
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 3 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 3 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 3
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3)));
+  }  
+#endif
+
+
+// ----------- thread starters for functions with 4 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 4 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 4 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 4
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+#endif
+
+// ----------- thread starters for functions with 5 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 5 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 5 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 5
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 6 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 6 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 6 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 6
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 7 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 7 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 7 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 7
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 8 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 8 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7, typename Arg8>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                           Arg6,Arg7,Arg8),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 8 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                              Arg6,Arg7,Arg8),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 8
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                              Arg6,Arg7,Arg8) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 9 arguments
+
+                                   /**
+                                    * Overload of the new_thread function for
+                                    * non-member or static member
+                                    * functions with 9 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Thread<RT>
+  new_thread (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                           Arg6,Arg7,Arg8,Arg9),
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8,
+             typename identity<Arg9>::type arg9)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr,
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8),
+                  internal::maybe_make_ref<Arg9>::act(arg9)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_thread
+                                   * function for member functions with
+                                   * 9 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                              Arg6,Arg7,Arg8,Arg9),
+             typename identity<C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8,
+             typename identity<Arg9>::type arg9)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8),
+                  internal::maybe_make_ref<Arg9>::act(arg9)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_thread function for
+                                   * const member functions with 9
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Thread<RT>
+  new_thread (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                              Arg6,Arg7,Arg8,Arg9) const,
+             typename identity<const C>::type &c,
+             typename identity<Arg1>::type arg1,
+             typename identity<Arg2>::type arg2,
+             typename identity<Arg3>::type arg3,
+             typename identity<Arg4>::type arg4,
+             typename identity<Arg5>::type arg5,
+             typename identity<Arg6>::type arg6,
+             typename identity<Arg7>::type arg7,
+             typename identity<Arg8>::type arg8,
+             typename identity<Arg9>::type arg9)
+  {
+    return
+      Thread<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                  internal::maybe_make_ref<Arg1>::act(arg1),
+                  internal::maybe_make_ref<Arg2>::act(arg2),
+                  internal::maybe_make_ref<Arg3>::act(arg3),
+                  internal::maybe_make_ref<Arg4>::act(arg4),
+                  internal::maybe_make_ref<Arg5>::act(arg5),
+                  internal::maybe_make_ref<Arg6>::act(arg6),
+                  internal::maybe_make_ref<Arg7>::act(arg7),
+                  internal::maybe_make_ref<Arg8>::act(arg8),
+                  internal::maybe_make_ref<Arg9>::act(arg9)));
+  }  
+#endif
+  
+// ------------------------ ThreadGroup -------------------------------------  
+
+                                   /**
+                                    * A container for thread
+                                    * objects. Allows to add new
+                                    * thread objects and wait for them
+                                    * all together. The thread objects
+                                    * need to have the same return
+                                    * value for the called function.
+                                    *
+                                    * @author Wolfgang Bangerth, 2003
+                                   * @ingroup threads
+                                    */
+  template <typename RT = void>
+  class ThreadGroup 
+  {
+    public:
+                                       /**
+                                        * Add another thread object to
+                                        * the collection.
+                                        */
+      ThreadGroup & operator += (const Thread<RT> &t)
+       {
+         threads.push_back (t);
+         return *this;
+       }
+
+                                       /**
+                                        * Wait for all threads in the
+                                        * collection to finish. It is
+                                        * not a problem if some of
+                                        * them have already been
+                                        * waited for, i.e. you may
+                                        * call this function more than
+                                        * once, and you can also add
+                                        * new thread objects between
+                                        * subsequent calls to this
+                                        * function if you want.
+                                        */
+      void join_all () const {
+        for (typename std::list<Thread<RT> >::const_iterator
+               t=threads.begin(); t!=threads.end(); ++t)
+          t->join ();
+      }
+    
+    private:
+                                       /**
+                                        * List of thread objects.
+                                        */
+      std::list<Thread<RT> > threads;
+  };
+  
+
+  template <typename> class Task;
+
+  
+  namespace internal 
+  {
+#if (DEAL_II_USE_MT == 1)
+
+    template <typename> class TaskDescriptor;
+
+    
+                                    /**
+                                     * The task class for TBB that is
+                                     * used by the TaskDescriptor
+                                     * class.
+                                     */
+    template <typename RT>
+    struct TaskEntryPoint : public tbb::task
+    {
+       TaskEntryPoint (TaskDescriptor<RT> &task_descriptor)
+                       :
+                       task_descriptor (task_descriptor)
+         {}
+
+       virtual tbb::task * execute ()
+         {
+                                            // call the function object
+                                            // and put the return value
+                                            // into the proper place
+           call (task_descriptor.function, task_descriptor.ret_val);
+
+                                            // indicate that the task
+                                            // has finished, both
+                                            // through the flag and
+                                            // through the mutex that
+                                            // was locked before we
+                                            // started and that now
+                                            // needs to be
+                                            // released. this may
+                                            // also wake up all
+                                            // threads that may be
+                                            // waiting for the task's
+                                            // demise by blocking on
+                                            // completion_mutex.acquire()
+                                            // in
+                                            // TaskDescriptor::join().
+           task_descriptor.task_is_done = true;
+           task_descriptor.completion_mutex.release ();
+               
+           return 0;
+         }
+
+                                        /**
+                                         * A reference to the descriptor
+                                         * object of this task.
+                                         */
+       TaskDescriptor<RT> &task_descriptor;
+    };  
+    
+                                     /**
+                                     * @internal
+                                      * Base class describing a
+                                      * task. This is the basic
+                                      * class abstracting the
+                                      * Threading Building Blocks
+                                      * implementation of tasks.
+                                      * It provides a mechanism
+                                      * to start a new task, as well
+                                      * as for joining it.
+                                     *
+                                     * Internally, the way things are
+                                     * implemented is that all Task<>
+                                     * objects keep a shared pointer
+                                     * to the task descriptor. When
+                                     * the last Task<> object goes
+                                     * out of scope, the destructor
+                                     * of the descriptor is
+                                     * called. Since tasks can not be
+                                     * abandoned, the destructor
+                                     * makes sure that the task is
+                                     * finished before it can
+                                     * continue to destroy the
+                                     * object.
+                                      *
+                                     * Note that unlike threads,
+                                     * tasks are not always started
+                                     * right away, and so the
+                                     * starting thread can't rely on
+                                     * the fact that the started task
+                                     * can copy things off the
+                                     * spawning thread's stack
+                                     * frame. As a consequence, the
+                                     * task description needs to
+                                     * include a way to store the
+                                     * function and its arguments
+                                     * that shall be run on the task.
+                                     *
+                                      * @author Wolfgang Bangerth, 2009
+                                      */
+    template <typename RT>
+    struct TaskDescriptor
+    {
+      private:
+                                        /**
+                                         * The function and its arguments
+                                         * that are to be run on the task.
+                                         */
+       std_cxx1x::function<RT ()> function;
+       
+                                         /**
+                                          * Variable holding the data the TBB
+                                          * needs to work with a task. Set by
+                                          * the queue_up_task() function. Note
+                                          * that the object behind this
+                                          * pointer will be deleted upon
+                                          * termination of the task, so we do
+                                          * not have to do so ourselves. In
+                                          * particular, if all objects with
+                                          * pointers to this task_description
+                                          * object go out of scope then no
+                                          * action is needed on our behalf.
+                                          */
+       tbb::task *task;
+
+                                        /**
+                                         * A place where the task will
+                                         * deposit its return value.
+                                         */
+        return_value<RT> ret_val;
+       
+                                        /**
+                                         * A flag indicating whether the task
+                                         * has terminated.
+                                         */
+       bool task_is_done;
+
+                                         /**
+                                          * Mutex used to indicate
+                                          * when the task is done. It
+                                          * is locked before the task
+                                          * is spawned; the join()
+                                          * function tries to acquire
+                                          * it, but that will fail
+                                          * unless the task has
+                                          * unlocked it, which it does
+                                          * upon completion.
+                                          */
+        mutable ThreadMutex completion_mutex;
+
+      public:
+
+                                        /**
+                                         * Constructor. Take the function to
+                                         * be run on this task as argument.
+                                         */
+       TaskDescriptor (const std_cxx1x::function<RT ()> &function);
+
+                                        /**
+                                         * Default
+                                         * constructor. Throws an
+                                         * exception since we want to
+                                         * queue a task immediately
+                                         * upon construction of these
+                                         * objects to make sure that
+                                         * each TaskDescriptor object
+                                         * corresponds to exactly one
+                                         * task.
+                                         */
+       TaskDescriptor ();
+       
+                                        /**
+                                         * Copy constructor. Throws
+                                         * an exception since we want
+                                         * to make sure that each
+                                         * TaskDescriptor object
+                                         * corresponds to exactly one
+                                         * task.
+                                         */
+       TaskDescriptor (const TaskDescriptor &);
+       
+                                         /**
+                                          * Destructor.
+                                          */
+        ~TaskDescriptor ();
+
+                                        /**
+                                         * Queue up the task to the
+                                         * scheduler. We need to do
+                                         * this in a separate
+                                         * function since we the new
+                                         * tasks needs to access
+                                         * objects from the current
+                                         * object and that can only
+                                         * reliably happen if the
+                                         * current object is
+                                         * completely constructed
+                                         * already.
+                                         */
+       void queue_task ();
+       
+                                         /**
+                                          * Join a task, i.e. wait
+                                          * for it to finish. This
+                                          * function can safely be
+                                          * called from different
+                                          * threads at the same time,
+                                          * and can also be called
+                                          * more than once.
+                                          */
+        void join ();
+
+
+       template <typename> friend struct TaskEntryPoint;
+       template <typename> friend struct Task;
+    };
+
+
+
+    template <typename RT>
+    inline
+    TaskDescriptor<RT>::TaskDescriptor (const std_cxx1x::function<RT ()> &function)
+                   :
+                   function (function),
+                   task_is_done (false)                    
+    {}
+
+
+    template <typename RT>
+    inline
+    void
+    TaskDescriptor<RT>::queue_task ()
+    {
+                                      // lock the mutex. it will
+                                      // become unlocked when the
+                                      // task is done
+      completion_mutex.acquire ();
+      
+                                      // use the pattern described in
+                                      // the TBB book on pages
+                                      // 230/231 ("Start a large task
+                                      // in parallel with the main
+                                      // program)
+      task = new (tbb::task::allocate_root()) tbb::empty_task;
+      task->set_ref_count (2);
+
+      tbb::task *worker = new (task->allocate_child()) TaskEntryPoint<RT>(*this);
+      task->spawn (*worker);
+    }
+
+
+
+    template <typename RT>
+    TaskDescriptor<RT>::TaskDescriptor ()
+    {
+      Assert (false, ExcInternalError());
+    }
+
+
+
+    template <typename RT>
+    TaskDescriptor<RT>::TaskDescriptor (const TaskDescriptor &)
+    {
+      Assert (false, ExcInternalError());
+    }
+    
+
+
+    template <typename RT>
+    inline
+    TaskDescriptor<RT>::~TaskDescriptor ()
+    {
+                                      // wait for the task to
+                                      // complete for sure
+      join ();
+
+                                      // now destroy the empty task
+                                      // structure. the book
+                                      // recommends to spawn it as
+                                      // well and let the scheduler
+                                      // destroy the object when
+                                      // done, but this has the
+                                      // disadvantage that the
+                                      // scheduler may not get to
+                                      // actually finishing the task
+                                      // before it goes out of scope
+                                      // (at the end of the program,
+                                      // or if a thread is done on
+                                      // which it was run) and then
+                                      // we would get a
+                                      // hard-to-decipher warning
+                                      // about unfinished tasks when
+                                      // the scheduler "goes out of
+                                      // the arena". rather, let's
+                                      // explicitly destroy the empty
+                                      // task object
+      Assert (task != 0, ExcInternalError());
+      task->wait_for_all ();
+      task->destroy (*task);
+    }
+
+    
+    template <typename RT>
+    inline
+    void
+    TaskDescriptor<RT>::join ()
+    {
+                                       // use Schmidt's double checking
+                                       // pattern: if thread has already
+                                       // indicated that it is done, then
+                                       // return immediately
+      if (task_is_done)
+       return;
+
+                                      // acquire the lock; this can
+                                      // only succeed when the task
+                                      // is done
+      completion_mutex.acquire ();
+
+                                      // release it again; at this
+                                      // point the task must have
+                                      // finished
+      completion_mutex.release ();
+      Assert (task_is_done == true, ExcInternalError());
+    }
+
+
+
+#else        // no threading enabled
+
+                                    /**
+                                     * A way to describe tasks. Since
+                                     * we are in non-MT mode at this
+                                     * place, things are a lot
+                                     * simpler than in MT mode.
+                                     */
+    template <typename RT>
+    struct TaskDescriptor
+    {
+                                        /**
+                                         * A place where the task will
+                                         * deposit its return value.
+                                         */
+        return_value<RT> ret_val;
+
+                                        /**
+                                         * Constructor. Call the
+                                         * given function and emplace
+                                         * the return value into the
+                                         * slot reserved for this
+                                         * purpose.
+                                         */
+       TaskDescriptor (const std_cxx1x::function<RT ()> &function)
+         {
+           call (function, ret_val);
+         }
+                                        /**
+                                         * Wait for the task to
+                                         * return. Since we are in
+                                         * non-MT mode here, there is
+                                         * nothing to do.
+                                         */
+       static void join () {}
+
+                                        /**
+                                         * Run the task. Since we are
+                                         * here in non-MT mode, there
+                                         * is nothing to do that the
+                                         * constructor hasn't already
+                                         * done.
+                                         */
+       static void queue_task () {}
+    };
+    
+#endif
+    
+  }
+
+  
+  template <typename RT = void>
+  class Task
+  {
+                                       /**
+                                        * Construct a task object
+                                        * given a function object to
+                                        * execute on the task.
+                                        */
+    public:
+      Task (const std_cxx1x::function<RT ()> &function_object)
+       {
+                                          // create a task descriptor and tell it
+                                          // to queue itself up with the scheduling
+                                          // system
+         task_descriptor =
+           std_cxx1x::shared_ptr<internal::TaskDescriptor<RT> >
+           (new internal::TaskDescriptor<RT>(function_object));
+         task_descriptor->queue_task ();
+       }
+
+                                       /**
+                                        * Default constructor. You
+                                        * can't do much with a task
+                                        * object constructed this way,
+                                        * except for assigning it a
+                                        * task object that holds
+                                        * data created by the
+                                        * <tt>spawn</tt> functions.
+                                        */
+      Task () {}
+
+                                       /**
+                                        * Join the task represented
+                                        * by this object, i.e. wait
+                                        * for it to finish. You can't
+                                        * call this function if you
+                                        * have used the default
+                                        * constructor of this class
+                                        * and have not assigned a
+                                        * task object to it.
+                                        */
+      void join () const
+       {
+         AssertThrow (task_descriptor != 0, ExcNoTask());
+         task_descriptor->join ();
+       }
+
+                                       /**
+                                        * Get the return value of the
+                                        * function of the
+                                        * task. Since this is only
+                                        * available once the task
+                                        * finishes, this implicitely
+                                        * also calls <tt>join()</tt>.
+                                        */
+      RT return_value ()
+       {
+         join ();
+         return task_descriptor->ret_val.get();
+       }
+
+
+                                       /**
+                                        * Check for equality of task
+                                        * objects. Since objects of
+                                        * this class store an implicit
+                                        * pointer to an object that
+                                        * exists exactly once for each
+                                        * task, the check is simply
+                                        * to compare these pointers.
+                                        */
+      bool operator == (const Task &t)
+       {
+         AssertThrow (task_descriptor != 0, ExcNoTask());
+         return task_descriptor == t.task_descriptor;
+       }
+
+                                      /** @addtogroup Exceptions
+                                       * @{ */
+      
+                                       /**
+                                        * Exception
+                                        */
+      DeclException0 (ExcNoTask);
+                                      //@}
+    private:
+                                       /**
+                                        * Shared pointer to the object
+                                        * representing the task. Boost's
+                                        * shared pointer implementation will
+                                        * make sure that that object lives as
+                                        * long as there is at least one
+                                        * subscriber to it.
+                                        */
+      std_cxx1x::shared_ptr<internal::TaskDescriptor<RT> > task_descriptor;
+  };
+
+
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * objects that can be converted to
+                                    * std_cxx1x::function<RT ()>, i.e. anything
+                                    * that can be called like a function
+                                    * object without arguments and returning
+                                    * an object of type RT (or void).
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT>
+  inline
+  Task<RT>
+  new_task (const std_cxx1x::function<RT ()> &function)
+  {
+    return Task<RT>(function);
+  }
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with no arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)())
+  {
+    return new_task<RT>(std_cxx1x::function<RT ()>(fun_ptr));
+  }
+  
+                                   /**
+                                    * Overload of the non-const new_task
+                                    * function for member functions with
+                                    * no arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(),
+           typename identity<C>::type &c)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                   /**
+                                    * Overload of the new_task function for
+                                    * const member functions with no
+                                    * arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)() const,
+           const typename identity<C>::type &c)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c)));
+  }
+#endif  
+
+
+
+// ----------- thread starters for unary functions
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 1 argument.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename Arg1>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1),
+           typename identity<Arg1>::type arg1)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 1 argument.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 1
+                                   * argument.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1)));
+  }
+#endif
+
+// ----------- thread starters for binary functions
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 2 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename Arg1, typename Arg2>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 2 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1, typename Arg2>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 2
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C, typename Arg1, typename Arg2>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2)));
+  }  
+#endif
+
+// ----------- thread starters for ternary functions
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 3 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 3 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 3
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3)));
+  }  
+#endif
+
+
+// ----------- thread starters for functions with 4 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 4 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 4 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 4
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3, typename Arg4>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4)));
+  }
+#endif
+
+// ----------- thread starters for functions with 5 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 5 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 5 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 5
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 6 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 6 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 6 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 6
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 7 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 7 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 7 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 7
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,Arg6,Arg7) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 8 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 8 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7, typename Arg8>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                         Arg6,Arg7,Arg8),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 8 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                            Arg6,Arg7,Arg8),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 8
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                            Arg6,Arg7,Arg8) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8)));
+  }
+#endif  
+
+// ----------- thread starters for functions with 9 arguments
+
+                                   /**
+                                    * Overload of the new_task function for
+                                    * non-member or static member
+                                    * functions with 9 arguments.
+                                    *
+                                   * @ingroup threads
+                                   */
+  template <typename RT,
+            typename Arg1, typename Arg2, typename Arg3,
+            typename Arg4, typename Arg5, typename Arg6,
+            typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Task<RT>
+  new_task (RT (*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                         Arg6,Arg7,Arg8,Arg9),
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8,
+           typename identity<Arg9>::type arg9)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr,
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8),
+                      internal::maybe_make_ref<Arg9>::act(arg9)));
+  }
+
+
+
+                                  /**
+                                   * Overload of the non-const new_task
+                                   * function for member functions with
+                                   * 9 arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                            Arg6,Arg7,Arg8,Arg9),
+           typename identity<C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8,
+           typename identity<Arg9>::type arg9)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::ref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8),
+                      internal::maybe_make_ref<Arg9>::act(arg9)));
+  }
+
+#ifndef DEAL_II_CONST_MEMBER_DEDUCTION_BUG
+                                  /**
+                                   * Overload of the new_task function for
+                                   * const member functions with 9
+                                   * arguments.
+                                   *
+                                   * @ingroup threads
+                                   */
+  template <typename RT, typename C,
+           typename Arg1, typename Arg2, typename Arg3,
+           typename Arg4, typename Arg5, typename Arg6,
+           typename Arg7, typename Arg8, typename Arg9>
+  inline
+  Task<RT>
+  new_task (RT (C::*fun_ptr)(Arg1,Arg2,Arg3,Arg4,Arg5,
+                            Arg6,Arg7,Arg8,Arg9) const,
+           typename identity<const C>::type &c,
+           typename identity<Arg1>::type arg1,
+           typename identity<Arg2>::type arg2,
+           typename identity<Arg3>::type arg3,
+           typename identity<Arg4>::type arg4,
+           typename identity<Arg5>::type arg5,
+           typename identity<Arg6>::type arg6,
+           typename identity<Arg7>::type arg7,
+           typename identity<Arg8>::type arg8,
+           typename identity<Arg9>::type arg9)
+  {
+    return
+      new_task<RT>
+      (std_cxx1x::bind(fun_ptr, std_cxx1x::cref(c),
+                      internal::maybe_make_ref<Arg1>::act(arg1),
+                      internal::maybe_make_ref<Arg2>::act(arg2),
+                      internal::maybe_make_ref<Arg3>::act(arg3),
+                      internal::maybe_make_ref<Arg4>::act(arg4),
+                      internal::maybe_make_ref<Arg5>::act(arg5),
+                      internal::maybe_make_ref<Arg6>::act(arg6),
+                      internal::maybe_make_ref<Arg7>::act(arg7),
+                      internal::maybe_make_ref<Arg8>::act(arg8),
+                      internal::maybe_make_ref<Arg9>::act(arg9)));
+  }  
+#endif
+
+
+// ------------------------ TaskGroup -------------------------------------  
+
+                                   /**
+                                    * A container for task
+                                    * objects. Allows to add new
+                                    * task objects and wait for them
+                                    * all together. The task objects
+                                    * need to have the same return
+                                    * value for the called function.
+                                    *
+                                    * @author Wolfgang Bangerth, 2003
+                                   * @ingroup tasks
+                                    */
+  template <typename RT = void>
+  class TaskGroup 
+  {
+    public:
+                                       /**
+                                        * Add another task object to
+                                        * the collection.
+                                        */
+      TaskGroup & operator += (const Task<RT> &t)
+       {
+         tasks.push_back (t);
+         return *this;
+       }
+
+                                       /**
+                                        * Wait for all tasks in the
+                                        * collection to finish. It is
+                                        * not a problem if some of
+                                        * them have already been
+                                        * waited for, i.e. you may
+                                        * call this function more than
+                                        * once, and you can also add
+                                        * new task objects between
+                                        * subsequent calls to this
+                                        * function if you want.
+                                        */
+      void join_all () const {
+        for (typename std::list<Task<RT> >::const_iterator
+               t=tasks.begin(); t!=tasks.end(); ++t)
+          t->join ();
+      }
+    
+    private:
+                                       /**
+                                        * List of task objects.
+                                        */
+      std::list<Task<RT> > tasks;
+  };
+  
+}   // end of implementation of namespace Threads
+
+/**
+ * @}
+ */
 
 
 //---------------------------------------------------------------------------
diff --git a/deal.II/base/include/base/work_stream.h b/deal.II/base/include/base/work_stream.h
new file mode 100644 (file)
index 0000000..ec6d5ea
--- /dev/null
@@ -0,0 +1,623 @@
+//---------------------------------------------------------------------------
+//    $Id$
+//    Version: $Name$
+//
+//    Copyright (C) 2008, 2009 by the deal.II authors
+//
+//    This file is subject to QPL and may not be  distributed
+//    without copyright and license information. Please refer
+//    to the file deal.II/doc/license.html for the  text  and
+//    further information on this license.
+//
+//---------------------------------------------------------------------------
+#ifndef __deal2__work_stream_h
+#define __deal2__work_stream_h
+
+
+#include <base/config.h>
+#include <base/multithread_info.h>
+#include <base/template_constraints.h>
+#include <base/std_cxx1x/function.h>
+#include <base/std_cxx1x/bind.h>
+
+#if DEAL_II_USE_MT == 1
+#  include <base/thread_management.h>
+#  include <tbb/pipeline.h>
+#endif
+
+#include <vector>
+#include <utility>
+
+
+DEAL_II_NAMESPACE_OPEN
+
+
+
+/**
+ * A class whose main template function supports running multiple
+ * threads each of which operates on a subset of the given range of
+ * objects. The class uses the Intel Threading Building Blocks (TBB)
+ * to load balance the individual subranges onto the available
+ * threads. For a lengthy discussion of the rationale of this class,
+ * see the @ref threads "Parallel computing with multiple processors"
+ * module.
+ *
+ * The class is built on the following premise: One frequently has some work
+ * that needs to be done on a sequence of objects; a prototypical example is
+ * assembling cell contributions to a system matrix or right hand side. In
+ * many such examples, part of the work can be done entirely independently and
+ * in parallel, possibly using several processor cores on a machine with
+ * shared memory. However, some other part of this work may need to be
+ * synchronised and be done in order. In the example of assembling a matrix,
+ * the computation of local contributions can be done entirely in parallel,
+ * but copying the the local contributions into the global matrix requires
+ * some care: First, several threads can't write at the same time, but need to
+ * synchronise writing using a mutex; secondly, we want the order in which
+ * local contributions are added to the global matrix to be always the same
+ * because floating point addition is not commutative and adding local
+ * contributions to the global matrix in different orders leads to subtly
+ * different results that can affect the number of iterations for iterative
+ * solvers as well as the round-off error in the solution in random
+ * ways. Consequently, we want to ensure that only one thread at a time writes
+ * into the global matrix, and that results are copied in a stable and
+ * reproducible order.
+ *
+ * This class implements a framework for this work model. It works with a
+ * stream of objects given by an iterator range, runs a worker function in
+ * parallel on all of these objects and then passes each object to a
+ * postprocessor function that runs sequentially and gets objects in exactly
+ * the order in which they appear in the input iterator range. None of the
+ * synchronisation work is exposed to the user of this class.
+ *
+ * Internally, the range given to the run() function of this class is split
+ * into a sequence of "items", which are then distributed according to some
+ * %internal algorithm onto the number of available threads. An item is an
+ * element of the range of iterators on which we are to operate; for example,
+ * for the purpose of assembling matrices or evaluating error indicators, an
+ * item could be a cell. The TBB library determines how many threads are
+ * created (typically as many as there are processor cores), but the number of
+ * items that may be active at any given time is specified by the argument to
+ * the constructor. It should be bigger or equal to the number of processor
+ * cores - the default is four times the number of cores on the current system.
+ *
+ * Items are created upon request by the TBB whenever one of the worker
+ * threads is idle or is expected to become idle. It is then handed off to a
+ * worker function, typically a member function of a main class. These worker
+ * functions are run in parallel on a number of threads, and there is no
+ * guarantee that they are asked to work on items in any particular order, in
+ * particular not necessarily in the order in which items are generated from
+ * the iterator range.
+ *
+ * Typically, worker functions need additional data, for example FEValues
+ * objects, input data vectors, etc, some of which can not be shared among
+ * threads. To this end, the run() function takes another template argument,
+ * ScratchData, which designates a type objects of which are stored with
+ * each item and which threads can use as private data without having to
+ * share them with other threads. The run() function takes an additional
+ * argument with an object of type ScratchData that is going to be copied
+ * for the arguments passed to each of the worker functions.
+ *
+ * In addition, worker functions store their results in objects of template type
+ * CopyData. These are then handed off to a separate function, called copier,
+ * that may use the stored results to transfer them into permanent
+ * storage. For example, it may copy the results of local contributions to a
+ * matrix computed by a worker function into the global matrix. In contrast to
+ * the worker function, however, only one instance of the copier is run at any
+ * given time; it can therefore safely copy local contributions into the
+ * global matrix without the need to lock the global object using a mutex or
+ * similar means. Furthermore, it is guaranteed that the copier is run with
+ * CopyData objects in the same order in which their associated items
+ * were created; consequently, even if worker threads may compute results in
+ * unspecified order, the copier always receives the results in exactly the
+ * same order as the items were created.
+ *
+ * Once an item is processed by the copier, it is deleted and the
+ * ScratchData and CopyData objects that were used in its computation
+ * are considered unused and may be re-used for the next invokation of
+ * the worker function, on this or another thread.
+ *
+ * This class only really works in parallel when multithread mode was selected
+ * during deal.II configuration. Otherwise it simply works on each item
+ * sequentially.
+ *
+ * @author Wolfgang Bangerth, 2007, 2008, 2009
+ */
+namespace WorkStream
+{
+
+#if DEAL_II_USE_MT == 1
+
+
+  namespace internal
+  {
+                                    /**
+                                     * A class that creates a sequence of
+                                     * items from a range of iterators.
+                                     */
+    template <typename Iterator,
+             typename ScratchData,
+             typename CopyData>
+    class IteratorRangeToItemStream : public tbb::filter
+    {
+      public:
+                                        /**
+                                         * A data type that we use to identify
+                                         * items to be worked on.
+                                         *
+                                         * The first element indicates an array
+                                         * of iterators to work on; the second
+                                         * the scratch space; the third an
+                                         * array of copy data spaces; and the
+                                         * last the number of elements to work
+                                         * on. This last argument is an integer
+                                         * between one and chunk_size. The
+                                         * arrays have a length of chunk_size.
+                                         */
+       typedef
+       std_cxx1x::tuple<std::vector<Iterator>,
+                        ScratchData*,
+                        std::vector<CopyData>,
+                        unsigned int>
+       ItemType;
+       
+
+                                        /**
+                                         * Constructor. Take an iterator
+                                         * range, the size of a buffer that
+                                         * can hold items, and the sample
+                                         * additional data object that will
+                                         * be passed to each worker and
+                                         * copier function invokation.
+                                         */
+       IteratorRangeToItemStream (const Iterator       &begin,
+                                  const Iterator       &end,
+                                  const unsigned int    buffer_size,
+                                  const unsigned int    chunk_size,
+                                  const ScratchData    &sample_scratch_data,
+                                  const CopyData       &sample_copy_data)
+                       :
+                       tbb::filter (/*is_serial=*/true),
+                       remaining_iterator_range (begin, end),
+                       ring_buffer (buffer_size),
+                       n_emitted_items (0),
+                       chunk_size (chunk_size)
+         {
+                                            // initialize copies of
+                                            // additional_data. since
+                                            // this is frequently
+                                            // expensive (creating
+                                            // FEValues objects etc) do
+                                            // that in parallel
+           Threads::TaskGroup<> tasks;
+           for (unsigned int i=0; i<ring_buffer.size(); ++i)
+             tasks += Threads::new_task (&IteratorRangeToItemStream::init_buffer_elements,
+                                         *this,
+                                         i,
+                                         std_cxx1x::cref(sample_scratch_data),
+                                         std_cxx1x::cref(sample_copy_data));
+           tasks.join_all ();
+         }
+
+                                        /**
+                                         * Destructor.
+                                         */
+       ~IteratorRangeToItemStream ()
+         {
+           for (unsigned int i=0; i<ring_buffer.size(); ++i)
+             delete std_cxx1x::get<1>(ring_buffer[i]);
+         }
+      
+                                        /**
+                                         * Create a item and return a
+                                         * pointer to it.
+                                         */
+       virtual void * operator () (void *)
+         {
+                                            // store the current
+                                            // position of the pointer
+           ItemType *current_item
+             = &ring_buffer[n_emitted_items % ring_buffer.size()];
+
+                                            // initialize the next item. it may
+                                            // consist of at most chunk_size
+                                            // elements
+           std_cxx1x::get<3>(*current_item) = 0;
+           while ((remaining_iterator_range.first !=
+                   remaining_iterator_range.second)
+                  &&
+                  (std_cxx1x::get<3>(*current_item) < chunk_size))
+             {
+               std_cxx1x::get<0>(*current_item)[std_cxx1x::get<3>(*current_item)]
+                 = remaining_iterator_range.first;
+
+               ++remaining_iterator_range.first;
+               ++std_cxx1x::get<3>(*current_item);
+             }
+
+           if (std_cxx1x::get<3>(*current_item) == 0)
+                                              // there were no items
+                                              // left. terminate the pipeline
+             return 0;
+           else
+             {
+               ++n_emitted_items;
+               return current_item;
+             }
+         }
+       
+      private:
+                                        /**
+                                         * The interval of iterators still to
+                                         * be worked on. This range will shrink
+                                         * over time.
+                                         */
+       std::pair<Iterator,Iterator> remaining_iterator_range;
+      
+                                        /**
+                                         * A ring buffer that will store items.
+                                         */
+       std::vector<ItemType>        ring_buffer;
+
+                                        /**
+                                         * Counter for the number of emitted
+                                         * items. Each item may consist of up
+                                         * to chunk_size iterator elements.
+                                         */
+       unsigned int                 n_emitted_items;
+
+                                        /**
+                                         * Number of elements of the
+                                         * iterator range that each
+                                         * thread should work on
+                                         * sequentially; a large number
+                                         * makes sure that each thread
+                                         * gets a significant amount of
+                                         * work before the next task
+                                         * switch happens, whereas a
+                                         * small number is better for
+                                         * load balancing.
+                                         */
+       const unsigned int           chunk_size;
+
+                                        /**
+                                         * Initialize the pointers and vector
+                                         * elements in the specified entry of
+                                         * the ring buffer.
+                                         */
+       void init_buffer_elements (const unsigned int element,
+                                  const ScratchData &sample_scratch_data,
+                                  const CopyData    &sample_copy_data)
+         {
+           Assert (std_cxx1x::get<1>(ring_buffer[element]) == 0,
+                   ExcInternalError());
+             
+           std_cxx1x::get<0>(ring_buffer[element])
+             .resize (chunk_size, remaining_iterator_range.second);
+           std_cxx1x::get<1>(ring_buffer[element])
+             = new ScratchData(sample_scratch_data);
+           std_cxx1x::get<2>(ring_buffer[element])
+             .resize (chunk_size, sample_copy_data);
+         }
+    };
+
+
+
+                                    /**
+                                     * A class that manages calling the
+                                     * worker function on a number of
+                                     * parallel threads. Note that it is, in
+                                     * the TBB notation, a filter that can
+                                     * run in parallel.
+                                     */
+    template <typename Iterator,
+             typename ScratchData,
+             typename CopyData>
+    class Worker : public tbb::filter
+    {
+      public:
+                                        /**
+                                         * Constructor. Takes a
+                                         * reference to the object on
+                                         * which we will operate as
+                                         * well as a pointer to the
+                                         * function that will do the
+                                         * assembly.
+                                         */
+       Worker (const std_cxx1x::function<void (const Iterator &,
+                                               ScratchData &,
+                                               CopyData &)> &worker)
+       :
+       tbb::filter (/* is_serial= */ false),
+       worker (worker)
+       {}
+
+
+                                        /**
+                                         * Work on an item.
+                                         */
+       void * operator () (void *item)
+         {
+                                            // first unpack the current item
+           typedef
+             typename IteratorRangeToItemStream<Iterator,ScratchData,CopyData>::ItemType
+             ItemType;
+           
+           ItemType *current_item = reinterpret_cast<ItemType*> (item);
+           
+                                            // then call the worker function on
+                                            // each element of the chunk we
+                                            // were given
+           for (unsigned int i=0; i<std_cxx1x::get<3>(*current_item); ++i)
+             worker (std_cxx1x::get<0>(*current_item)[i],
+                     *std_cxx1x::get<1>(*current_item),
+                     std_cxx1x::get<2>(*current_item)[i]);
+           
+                                            // then return the original pointer
+                                            // to the now modified object
+           return item;
+         }
+       
+
+      private:
+                                        /**
+                                         * Pointer to the function
+                                         * that does the assembling
+                                         * on the sequence of cells.
+                                         */
+       const std_cxx1x::function<void (const Iterator &,
+                                       ScratchData &,
+                                       CopyData &)> worker;
+    };
+
+
+
+                                    /**
+                                     * A class that manages calling the
+                                     * copier function. Note that it is, in
+                                     * the TBB notation, a filter that runs
+                                     * sequentially, ensuring that all items
+                                     * are copied in the same order in which
+                                     * they are created.
+                                     */
+    template <typename Iterator,
+             typename ScratchData,
+             typename CopyData>
+    class Copier : public tbb::filter
+    {
+      public:
+                                        /**
+                                         * Constructor. Takes a
+                                         * reference to the object on
+                                         * which we will operate as
+                                         * well as a pointer to the
+                                         * function that will do the
+                                         * copying from the
+                                         * additional data object to
+                                         * the global matrix or
+                                         * similar.
+                                         */
+       Copier (const std_cxx1x::function<void (const CopyData &)> &copier)
+                       :
+                       tbb::filter (/* is_serial= */ true),
+                       copier (copier)
+         {}
+
+
+                                        /**
+                                         * Work on a single item.
+                                         */
+       void * operator () (void *item)
+         {
+                                            // first unpack the current item
+           typedef
+             typename IteratorRangeToItemStream<Iterator,ScratchData,CopyData>::ItemType
+             ItemType;
+           
+           ItemType *current_item = reinterpret_cast<ItemType*> (item);
+
+                                            // initiate copying data
+           for (unsigned int i=0; i<std_cxx1x::get<3>(*current_item); ++i)
+             copier (std_cxx1x::get<2>(*current_item)[i]);
+
+                                            // return an invalid
+                                            // item since we are at
+                                            // the end of the
+                                            // pipeline
+           return 0;
+         }
+       
+
+      private:
+                                        /**
+                                         * Pointer to the function
+                                         * that does the copying of
+                                         * data.
+                                         */
+       const std_cxx1x::function<void (const CopyData &)> copier;
+    };
+    
+  }
+  
+
+#endif // DEAL_II_USE_MT
+
+
+
+                                  /**
+                                   * This is the main function of the
+                                   * WorkStream concept, doing work as
+                                   * described in the introduction to this
+                                   * namespace.
+                                   *
+                                   * This is the function that can be used
+                                   * for worker and copier objects that are
+                                   * either pointers to non-member
+                                   * functions or objects that allow to be
+                                   * called with an operator(), for example
+                                   * objects created by std::bind.
+                                   *
+                                   * The argument passed as @p end must be
+                                   * convertible to the same type as
+                                   * @p begin, but doesn't have to be of the
+                                   * same type itself. This allows to write
+                                   * code like
+                                   * <code>WorkStream().run(dof_handler.begin_active(),
+                                   * dof_handler.end(), ...</code> where
+                                   * the first is of type
+                                   * DoFHandler::active_cell_iterator
+                                   * whereas the second is of type
+                                   * DoFHandler::raw_cell_iterator.
+                                   *
+                                   * The @p queue_length argument indicates
+                                   * the number of items that can be live
+                                   * at any given time. Each item consists
+                                   * of @p chunk_size elements of the input
+                                   * stream that will be worked on by the
+                                   * worker and copier functions one after
+                                   * the other on the same thread.
+                                   */
+  template <typename Worker,
+           typename Copier,
+           typename Iterator,
+           typename ScratchData,
+           typename CopyData>
+  void
+  run (const Iterator                          &begin,
+       const typename identity<Iterator>::type &end,
+       Worker                                   worker,
+       Copier                                   copier,
+       const ScratchData                       &sample_scratch_data,
+       const CopyData                          &sample_copy_data,
+       const unsigned int queue_length = 2*multithread_info.n_default_threads,
+       const unsigned int chunk_size = 8)
+  {
+    Assert (queue_length > 0,
+           ExcMessage ("The queue length must be at least one, and preferably "
+                       "larger than the number of processors on this system."));
+    Assert (chunk_size > 0,
+           ExcMessage ("The chunk_size must be at least one."));  
+
+                                    // if no work then skip. (only use
+                                    // operator!= for iterators since we may
+                                    // not have an equality comparison
+                                    // operator)
+    if (!(begin != end))
+      return;
+    
+#if DEAL_II_USE_MT == 1  
+                                    // create the three stages of the
+                                    // pipeline
+    internal::IteratorRangeToItemStream<Iterator,ScratchData,CopyData>
+      iterator_range_to_item_stream (begin, end,
+                                    queue_length * chunk_size,
+                                    chunk_size,
+                                    sample_scratch_data,
+                                    sample_copy_data);
+
+    internal::Worker<Iterator, ScratchData, CopyData> worker_filter (worker);
+    internal::Copier<Iterator, ScratchData, CopyData> copier_filter (copier);
+
+                                    // now create a pipeline from
+                                    // these stages
+    tbb::pipeline assembly_line;
+    assembly_line.add_filter (iterator_range_to_item_stream);
+    assembly_line.add_filter (worker_filter);
+    assembly_line.add_filter (copier_filter);
+
+                                    // and run it
+    assembly_line.run (queue_length);    
+
+    assembly_line.clear ();
+
+#else
+
+                                    // need to copy the sample since it is
+                                    // marked const
+    ScratchData scratch_data = sample_scratch_data;
+    CopyData    copy_data    = sample_copy_data;
+    
+    for (Iterator i=begin; i!=end; ++i)
+      {
+       worker (i, scratch_data, copy_data);
+       copier (copy_data);
+      }  
+#endif
+  }
+
+
+
+                                  /**
+                                   * This is the main function of the
+                                   * WorkStream concept, doing work as
+                                   * described in the introduction to this
+                                   * namespace.
+                                   *
+                                   * This is the function that can be
+                                   * used for worker and copier functions
+                                   * that are member functions of a class.
+                                   *
+                                   * The argument passed as @p end must be
+                                   * convertible to the same type as
+                                   * @p begin, but doesn't have to be of the
+                                   * same type itself. This allows to write
+                                   * code like
+                                   * <code>WorkStream().run(dof_handler.begin_active(),
+                                   * dof_handler.end(), ...</code> where
+                                   * the first is of type
+                                   * DoFHandler::active_cell_iterator
+                                   * whereas the second is of type
+                                   * DoFHandler::raw_cell_iterator.
+                                   *
+                                   * The @p queue_length argument indicates
+                                   * the number of items that can be live
+                                   * at any given time. Each item consists
+                                   * of @p chunk_size elements of the input
+                                   * stream that will be worked on by the
+                                   * worker and copier functions one after
+                                   * the other on the same thread.
+                                   */
+  template <typename MainClass,
+           typename Iterator,
+           typename ScratchData,
+           typename CopyData>
+  void
+  run (const Iterator                          &begin,
+       const typename identity<Iterator>::type &end,
+       MainClass                               &main_object,
+       void (MainClass::*worker) (const Iterator &,
+                                 ScratchData &,
+                                 CopyData &),
+       void (MainClass::*copier) (const CopyData &),
+       const ScratchData                    &sample_scratch_data,
+       const CopyData                       &sample_copy_data,
+       const unsigned int queue_length = 2*multithread_info.n_default_threads,
+       const unsigned int chunk_size = 8)
+  {
+                                    // forward to the other function
+    run (begin, end,
+        std_cxx1x::bind (worker,
+                         std_cxx1x::ref (main_object),
+                         _1, _2, _3),
+        std_cxx1x::bind (copier,
+                         std_cxx1x::ref (main_object),
+                         _1),
+        sample_scratch_data,
+        sample_copy_data,
+        queue_length,
+        chunk_size);
+  }
+
+}
+
+
+
+
+DEAL_II_NAMESPACE_CLOSE
+
+
+
+
+//----------------------------   work_stream.h     ---------------------------
+// end of #ifndef __deal2__work_stream_h
+#endif
+//----------------------------   work_stream.h     ---------------------------
index 71c166b6094a1894b7ddca1d744266fb4d9f762b..eb3cd5e21c7b997d6a6507f1727bbb7104c42b96 100644 (file)
@@ -3104,14 +3104,14 @@ void DataOutBase::write_gmv (const std::vector<Patch<dim,spacedim> > &patches,
                                   // this copying of data vectors can
                                   // be done while we already output
                                   // the vertices, so do this on a
-                                  // separate thread and when wanting
+                                  // separate task and when wanting
                                   // to write out the data, we wait
-                                  // for that thread to finish
+                                  // for that task to finish
   Table<2,double> data_vectors (n_data_sets, n_nodes);
   void (*fun_ptr) (const std::vector<Patch<dim,spacedim> > &,
                   Table<2,double> &)
     = &DataOutBase::template write_gmv_reorder_data_vectors<dim,spacedim>;
-  Threads::Thread<> reorder_thread = Threads::spawn (fun_ptr)(patches, data_vectors);
+  Threads::Task<> reorder_task = Threads::new_task (fun_ptr, patches, data_vectors);
 
                                   ///////////////////////////////
                                   // first make up a list of used
@@ -3149,7 +3149,7 @@ void DataOutBase::write_gmv (const std::vector<Patch<dim,spacedim> > &patches,
                                   // now write the data vectors to
                                   // @p{out} first make sure that all
                                   // data is in place
-  reorder_thread.join ();
+  reorder_task.join ();
 
                                   // then write data.
                                   // the '1' means: node data (as opposed
@@ -3286,16 +3286,16 @@ void DataOutBase::write_tecplot (const std::vector<Patch<dim,spacedim> > &patche
                                    // this copying of data vectors can
                                    // be done while we already output
                                    // the vertices, so do this on a
-                                   // separate thread and when wanting
+                                   // separate task and when wanting
                                    // to write out the data, we wait
-                                   // for that thread to finish
+                                   // for that task to finish
   
   Table<2,double> data_vectors (n_data_sets, n_nodes);
 
   void (*fun_ptr) (const std::vector<Patch<dim,spacedim> > &,
                    Table<2,double> &)
     = &DataOutBase::template write_gmv_reorder_data_vectors<dim,spacedim>;
-  Threads::Thread<> reorder_thread = Threads::spawn (fun_ptr)(patches, data_vectors);
+  Threads::Task<> reorder_task = Threads::new_task (fun_ptr, patches, data_vectors);
 
                                    ///////////////////////////////
                                    // first make up a list of used
@@ -3317,7 +3317,7 @@ void DataOutBase::write_tecplot (const std::vector<Patch<dim,spacedim> > &patche
                                    // now write the data vectors to
                                    // @p{out} first make sure that all
                                    // data is in place
-  reorder_thread.join ();
+  reorder_task.join ();
 
                                    // then write data.
   for (unsigned int data_set=0; data_set<n_data_sets; ++data_set)
@@ -3538,15 +3538,15 @@ void DataOutBase::write_tecplot_binary (const std::vector<Patch<dim,spacedim> >
                                   // this copying of data vectors can
                                   // be done while we already output
                                   // the vertices, so do this on a
-                                  // separate thread and when wanting
+                                  // separate task and when wanting
                                   // to write out the data, we wait
-                                  // for that thread to finish  
+                                  // for that task to finish  
   Table<2,double> data_vectors (n_data_sets, n_nodes);
 
   void (*fun_ptr) (const std::vector<Patch<dim,spacedim> > &,
                    Table<2,double> &)
     = &DataOutBase::template write_gmv_reorder_data_vectors<dim,spacedim>;
-  Threads::Thread<> reorder_thread = Threads::spawn (fun_ptr)(patches, data_vectors);
+  Threads::Task<> reorder_task = Threads::new_task (fun_ptr, patches, data_vectors);
   
                                   ///////////////////////////////
                                   // first make up a list of used
@@ -3619,7 +3619,7 @@ void DataOutBase::write_tecplot_binary (const std::vector<Patch<dim,spacedim> >
                                   ///////////////////////////////////////
                                   // data output.
                                   //
-  reorder_thread.join ();
+  reorder_task.join ();
 
                                   // then write data.
   for (unsigned int data_set=0; data_set<n_data_sets; ++data_set)
@@ -3823,15 +3823,15 @@ DataOutBase::write_vtk (const std::vector<Patch<dim,spacedim> > &patches,
                                   // this copying of data vectors can
                                   // be done while we already output
                                   // the vertices, so do this on a
-                                  // separate thread and when wanting
+                                  // separate task and when wanting
                                   // to write out the data, we wait
-                                  // for that thread to finish
+                                  // for that task to finish
   Table<2,double> data_vectors (n_data_sets, n_nodes);
 
   void (*fun_ptr) (const std::vector<Patch<dim,spacedim> > &,
                   Table<2,double> &)
     = &DataOutBase::template write_gmv_reorder_data_vectors<dim,spacedim>;
-  Threads::Thread<> reorder_thread = Threads::spawn (fun_ptr)(patches, data_vectors);
+  Threads::Task<> reorder_task = Threads::new_task (fun_ptr, patches, data_vectors);
 
                                   ///////////////////////////////
                                   // first make up a list of used
@@ -3863,7 +3863,7 @@ DataOutBase::write_vtk (const std::vector<Patch<dim,spacedim> > &patches,
                                   // now write the data vectors to
                                   // @p{out} first make sure that all
                                   // data is in place
-  reorder_thread.join ();
+  reorder_task.join ();
 
                                   // then write data.  the
                                   // 'POINT_DATA' means: node data
index a92ff3ec79b9180a770cec5d59eea7490b57c120..b6a9ba31a072fc01bcf04b4c28cd5c019d006b25 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -971,7 +971,6 @@ QGaussLogR<1>::QGaussLogR(const unsigned int n,
            this->quadrature_points[j] = quad.point(i)*fraction;
            this->weights[j] = -std::log(alpha/fraction)*quad.weight(i)*fraction;
        }
-
        // In case we need the second quadrature as well, do it now.
        if(fraction != 1) {
            this->quadrature_points[i+n] = quad2.point(i)*(1-fraction)+Point<1>(fraction);
index fa9ef7fa178d8f512cc6fc8722772384c922bbf0..b3669671ac1e53eee0b58f4f3e3b8e988aa335ba 100644 (file)
@@ -250,8 +250,17 @@ namespace Threads
          return_values[i].second = end;
       };
     return return_values;
-  }
-  
+  }  
 }   // end namespace Thread
 
+
+
+#if DEAL_II_USE_MT == 1
+/**
+ * A scheduler for tasks. 
+ */
+tbb::task_scheduler_init scheduler;
+#endif
+
+
 DEAL_II_NAMESPACE_CLOSE
index fd60f055c99eda5cfb5450fb10916669aa292147..9ad8dc73b17cd27de88f0cd7b2b1102f37b015eb 100644 (file)
@@ -38,8 +38,7 @@ F77LIBS              = @F77LIBS@
 LIBS                 = @LIBS@
 LIBDIR               = $D/lib
 PERL                 = @PERL@
-enable-multithreading= @enablemultithreading@
-with-multithreading  = @withmultithreading@
+enable-threads       = @enablethreads@
 enable-shared        = @enableshared@
 enable-parser        = @enableparser@
 DEAL_II_VERSION      = @PACKAGE_VERSION@
@@ -221,12 +220,20 @@ ifeq ($(USE_CONTRIB_HSL),yes)
   lib-hsl = $(LIBDIR)/libhsl$(lib-suffix) 
 endif
 
+# need to link with libtbb if we're using threads
+ifeq ($(enable-threads),yes)
+  lib-contrib-tbb.g = $D/lib/libtbb_debug$(lib-suffix)
+  lib-contrib-tbb.o = $D/lib/libtbb$(lib-suffix)
+endif
+
 
 # set paths to all the libraries we need:
 lib-base.o     = $(LIBDIR)/libbase$(lib-suffix) \
-                $(lib-contrib-trilinos)
+                $(lib-contrib-trilinos) \
+                $(lib-contrib-tbb.o)
 lib-base.g     = $(LIBDIR)/libbase.g$(lib-suffix) \
-                $(lib-contrib-trilinos)
+                $(lib-contrib-trilinos) \
+                $(lib-contrib-tbb.g)
 
 lib-lac.o      = $(LIBDIR)/liblac$(lib-suffix) \
                 $(lib-hsl) \
@@ -295,6 +302,9 @@ ifeq ($(USE_CONTRIB_METIS),yes)
   INCLUDE += -I$(include-path-metis)
 endif
 
+ifeq ($(enable-threads),yes)
+  INCLUDE += -I$(shell echo $D/contrib/tbb/tbb*/include)
+endif
 
 # compiler flags for debug and optimized mode
 CXXFLAGS.g   = @DEFS@ @CXXFLAGSG@  $(INCLUDE)
@@ -306,6 +316,12 @@ F77FLAGS.o   = @DEFS@ @F77FLAGSO@  $(INCLUDE)
 # compile flags for C compiler
 CFLAGS = @CFLAGS@
 
+# if in debug mode, add TBB assertions
+ifeq ($(enable-threads),yes)
+  CXXFLAGS.g += -DTBB_DO_ASSERT=1
+endif
+
+
 # PETSc wants to see a whole lot of other flags being passed...
 ifeq ($(USE_CONTRIB_PETSC),yes)
   # set PETSC_DIR and PETSC_ARCH to be used in variables file
@@ -324,3 +340,8 @@ ifeq ($(USE_CONTRIB_PETSC),yes)
   CXXFLAGS.o += $(OCXX_PETSCFLAGS)
 endif
 
+ifneq ($(enable-threads),no)
+  MT = MT
+else
+  MT = ==
+endif
index 13abec560dc0d6e485da00d89b529076f586e402..ebfeba3ca8715ac3bfc3b826b8d5da84f1f3d031 100644 (file)
@@ -11,7 +11,7 @@ include $D/common/Make.global_options
 # Targets for compilation
 ############################################################
 
-ifneq ($(with-multithreading),no)
+ifneq ($(enable-threads),no)
   MT = MT
 else
   MT = ==
index 1cd37a3fc59586a96bca5c35c18ad77aa4ad5c1a..f4be092e03cd73eb0b7a97554e5436a7c7c30790 100755 (executable)
@@ -691,8 +691,7 @@ SHLIBLD
 SHLIBFLAGS
 CC_VERSION
 CFLAGSPIC
-enablemultithreading
-withmultithreading
+enablethreads
 CXXCPP
 GREP
 EGREP
@@ -1347,8 +1346,7 @@ if test -n "$ac_init_help"; then
 Optional Features:
   --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
   --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
-  --enable-multithreading Set compiler flags to allow for multithreaded
-                          programs
+  --enable-threads  Use multiple threads inside deal.II
   --enable-shared         Set compiler flags to generate shared libraries
   --enable-parser         While switched on by default, this option allows to
                           switch off support for the function parser in the
@@ -1359,9 +1357,6 @@ Optional Features:
 Optional Packages:
   --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
   --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
-  --with-multithreading=name
-                          If name==posix, or no name given, then use POSIX
-                         threads.
   --with-boost=/path/to/boost
                           Use an installed boost library instead of the
                          contributed one. The optional argument points to the
@@ -4588,7 +4583,7 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
     fi
 
         case "$target" in
-                        *apple-darwin*)
+                                                      *apple-darwin*)
         OLD_CXXFLAGS="$CXXFLAGS"
         CXXFLAGS=-Wno-long-double
 
@@ -4644,6 +4639,13 @@ fi
 
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 
+        if test "`/usr/sbin/sysctl -n hw.optional.x86_64`" = "1" ; then
+          CXXFLAGS="$CXXFLAGS -m64"
+          CXXFLAGSG="$CXXFLAGSG -m64"
+          CXXFLAGSO="$CXXFLAGSO -m64"
+          LDFLAGS="$LDFLAGS -m64"
+        fi
+
         CXXFLAGS="${OLD_CXXFLAGS}"
         ;;
 
@@ -5426,6 +5428,12 @@ echo "${ECHO_T}Unrecognized compiler -- still trying" >&6; }
         CFLAGSPIC=
         ;;
 
+      *apple-darwin*)
+                                        if test "`/usr/sbin/sysctl -n hw.optional.x86_64`" = "1" ; then
+          CFLAGS="$CFLAGS -m64"
+        fi
+        ;;
+
       *)
        CFLAGSPIC="-fPIC"
        ;;
@@ -5619,16 +5627,35 @@ rm -f core conftest.err conftest.$ac_objext conftest_ipa8_conftest.oo \
 
 
 
-  # Check whether --enable-multithreading was given.
-if test "${enable_multithreading+set}" = set; then
-  enableval=$enable_multithreading; enablemultithreading="$enableval"
+  # Check whether --enable-threads was given.
+if test "${enable_threads+set}" = set; then
+  enableval=$enable_threads;
+      enablethreads="$enableval"
+
 else
-  enablemultithreading=no
+
+                  case "$target" in
+        *cygwin* )
+          enablethreads=no
+         ;;
+
+       * )
+          enablethreads=yes
+          ;;
+      esac
+
 fi
 
 
+  if test "$enablethreads" = yes ; then
+        case "$target" in
+      *cygwin* )
+        { { echo "$as_me:$LINENO: error: Multithreading is not supported on CygWin" >&5
+echo "$as_me: error: Multithreading is not supported on CygWin" >&2;}
+   { (exit 1); exit 1; }; }
+        ;;
+    esac
 
-  if test "$enablemultithreading" = yes ; then
     if test "$GXX" = yes ; then
 
   case "$target" in
@@ -5789,97 +5816,8 @@ echo "$as_me: error: No threading compiler options for this C++ compiler
            ;;
       esac
     fi
-  fi
-
-
-  if test "$enablemultithreading" = yes ; then
-    case "$GXX_VERSION" in
-      gcc*)
-       { echo "$as_me:$LINENO: checking for only partly bracketed mutex initializer" >&5
-echo $ECHO_N "checking for only partly bracketed mutex initializer... $ECHO_C" >&6; }
-       ac_ext=cpp
-ac_cpp='$CXXCPP $CPPFLAGS'
-ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
-ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
-ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
-
-       CXXFLAGS="$CXXFLAGSG -Werror"
-       cat >conftest.$ac_ext <<_ACEOF
-/* confdefs.h.  */
-_ACEOF
-cat confdefs.h >>conftest.$ac_ext
-cat >>conftest.$ac_ext <<_ACEOF
-/* end confdefs.h.  */
-
-#      include <vector>
-
-int
-main ()
-{
-;
-  ;
-  return 0;
-}
-_ACEOF
-rm -f conftest.$ac_objext
-if { (ac_try="$ac_compile"
-case "(($ac_try" in
-  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
-  *) ac_try_echo=$ac_try;;
-esac
-eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
-  (eval "$ac_compile") 2>conftest.er1
-  ac_status=$?
-  grep -v '^ *+' conftest.er1 >conftest.err
-  rm -f conftest.er1
-  cat conftest.err >&5
-  echo "$as_me:$LINENO: \$? = $ac_status" >&5
-  (exit $ac_status); } && {
-        test -z "$ac_cxx_werror_flag" ||
-        test ! -s conftest.err
-       } && test -s conftest.$ac_objext; then
-
-         { echo "$as_me:$LINENO: result: no" >&5
-echo "${ECHO_T}no" >&6; }
-
-else
-  echo "$as_me: failed program was:" >&5
-sed 's/^/| /' conftest.$ac_ext >&5
-
-
-         { echo "$as_me:$LINENO: result: yes" >&5
-echo "${ECHO_T}yes" >&6; }
-         CXXFLAGSG="$CXXFLAGSG -Wno-missing-braces"
-         CXXFLAGSO="$CXXFLAGSO -Wno-missing-braces"
-
-fi
-
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-       ;;
-      *)
-        ;;
-    esac
-  fi
-
-
 
 
-
-# Check whether --with-multithreading was given.
-if test "${with_multithreading+set}" = set; then
-  withval=$with_multithreading; withmultithreading="$withval"
-else
-  withmultithreading=no
-fi
-
-
-  if test "x$withmultithreading" = "xyes" ; then
-    withmultithreading=posix
-  fi
-
-  if test "x$withmultithreading" != "xno" ; then
-    if test "x$withmultithreading" = "xposix" ; then
-
   { echo "$as_me:$LINENO: checking for posix thread functions" >&5
 echo $ECHO_N "checking for posix thread functions... $ECHO_C" >&6; }
   ac_ext=cpp
@@ -6154,12 +6092,11 @@ cat >>confdefs.h <<\_ACEOF
 #define DEAL_II_USE_MT_POSIX 1
 _ACEOF
 
-    else
-      { { echo "$as_me:$LINENO: error: Invalid flag for --with-multithreading" >&5
-echo "$as_me: error: Invalid flag for --with-multithreading" >&2;}
-   { (exit 1); exit 1; }; }
-    fi
 
+            LDFLAGS="$LDFLAGS -ldl"
+  fi
+
+  if test "x$enablethreads" != "xno" ; then
     DEAL_II_USE_MT_VAL=1
   else
     DEAL_II_USE_MT_VAL=0
@@ -6172,6 +6109,76 @@ _ACEOF
 
 
 
+  if test "$enablethreads" = yes ; then
+    case "$GXX_VERSION" in
+      gcc*)
+       { echo "$as_me:$LINENO: checking for only partly bracketed mutex initializer" >&5
+echo $ECHO_N "checking for only partly bracketed mutex initializer... $ECHO_C" >&6; }
+       ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+       CXXFLAGS="$CXXFLAGSG -Werror"
+       cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+#      include <vector>
+
+int
+main ()
+{
+;
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+        test -z "$ac_cxx_werror_flag" ||
+        test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+
+         { echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6; }
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+
+         { echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6; }
+         CXXFLAGSG="$CXXFLAGSG -Wno-missing-braces"
+         CXXFLAGSO="$CXXFLAGSO -Wno-missing-braces"
+
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+       ;;
+      *)
+        ;;
+    esac
+  fi
+
+
 
 
 
@@ -8082,6 +8089,80 @@ sed 's/^/| /' conftest.$ac_ext >&5
       { echo "$as_me:$LINENO: result: no" >&5
 echo "${ECHO_T}no" >&6; }
 
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+
+  { echo "$as_me:$LINENO: checking for const member deduction bug" >&5
+echo $ECHO_N "checking for const member deduction bug... $ECHO_C" >&6; }
+  ac_ext=cpp
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='$CXX -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext >&5'
+ac_link='$CXX -o conftest$ac_exeext $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5'
+ac_compiler_gnu=$ac_cv_cxx_compiler_gnu
+
+  CXXFLAGS="$CXXFLAGSG"
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+      template <typename T> struct identity { typedef T type; };
+
+      template <typename C> void new_thread (void (C::*fun_ptr)(),
+                                      typename identity<C>::type &c);
+      template <typename C> void new_thread (void (C::*fun_ptr)() const,
+                                      const typename identity<C>::type &c);
+      struct X { void f() const; };
+
+int
+main ()
+{
+
+      X x;
+      new_thread (&X::f, x);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval "echo \"\$as_me:$LINENO: $ac_try_echo\"") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+        test -z "$ac_cxx_werror_flag" ||
+        test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+
+      { echo "$as_me:$LINENO: result: no" >&5
+echo "${ECHO_T}no" >&6; }
+
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+
+      { echo "$as_me:$LINENO: result: yes" >&5
+echo "${ECHO_T}yes" >&6; }
+
+cat >>confdefs.h <<\_ACEOF
+#define DEAL_II_CONST_MEMBER_DEDUCTION_BUG 1
+_ACEOF
+
+
 fi
 
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
@@ -12214,7 +12295,7 @@ if test "x$enableshared" = "xyes" ; then
 
 
     OLD_LDFLAGS="$LDFLAGS"
-  LDFLAGS="-Wl,-soname,libbase.so.6.2.1 $LDFLAGS $SHLIBFLAGS"
+  LDFLAGS="-Wl,-soname,libbase.so.6.2.1 $LDFLAGS $LDFLAGSPIC $SHLIBFLAGS"
   { echo "$as_me:$LINENO: checking whether compiler understands option -Wl,-soname" >&5
 echo $ECHO_N "checking whether compiler understands option -Wl,-soname... $ECHO_C" >&6; }
   cat >conftest.$ac_ext <<_ACEOF
@@ -17536,8 +17617,7 @@ SHLIBLD!$SHLIBLD$ac_delim
 SHLIBFLAGS!$SHLIBFLAGS$ac_delim
 CC_VERSION!$CC_VERSION$ac_delim
 CFLAGSPIC!$CFLAGSPIC$ac_delim
-enablemultithreading!$enablemultithreading$ac_delim
-withmultithreading!$withmultithreading$ac_delim
+enablethreads!$enablethreads$ac_delim
 CXXCPP!$CXXCPP$ac_delim
 GREP!$GREP$ac_delim
 EGREP!$EGREP$ac_delim
@@ -17560,6 +17640,7 @@ DEAL_II_DISABLE_PARSER!$DEAL_II_DISABLE_PARSER$ac_delim
 DEAL_II_EXPAND_PETSC_VECTOR!$DEAL_II_EXPAND_PETSC_VECTOR$ac_delim
 DEAL_II_EXPAND_PETSC_BLOCKVECTOR!$DEAL_II_EXPAND_PETSC_BLOCKVECTOR$ac_delim
 USE_CONTRIB_PETSC!$USE_CONTRIB_PETSC$ac_delim
+DEAL_II_PETSC_DIR!$DEAL_II_PETSC_DIR$ac_delim
 _ACEOF
 
   if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 97; then
@@ -17601,7 +17682,6 @@ _ACEOF
 ac_delim='%!_!# '
 for ac_last_try in false false false false false :; do
   cat >conf$$subs.sed <<_ACEOF
-DEAL_II_PETSC_DIR!$DEAL_II_PETSC_DIR$ac_delim
 DEAL_II_PETSC_ARCH!$DEAL_II_PETSC_ARCH$ac_delim
 DEAL_II_PETSC_VERSION_MAJOR!$DEAL_II_PETSC_VERSION_MAJOR$ac_delim
 DEAL_II_PETSC_VERSION_MINOR!$DEAL_II_PETSC_VERSION_MINOR$ac_delim
@@ -17647,7 +17727,7 @@ LIBOBJS!$LIBOBJS$ac_delim
 LTLIBOBJS!$LTLIBOBJS$ac_delim
 _ACEOF
 
-  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 44; then
+  if test `sed -n "s/.*$ac_delim\$/X/p" conf$$subs.sed | grep -c X` = 43; then
     break
   elif $ac_last_try; then
     { { echo "$as_me:$LINENO: error: could not make $CONFIG_STATUS" >&5
index e958c5f49e17f3f950818a02f6b11b5e895ca01c..484e7dcef3e07b9adc3165288588c5c096c78a9e 100644 (file)
@@ -145,13 +145,8 @@ dnl Test whether multithreading support is requested. This
 dnl does not tell deal.II to actually use it, but the
 dnl compiler flags are set to allow for it.
 DEAL_II_CHECK_MULTITHREADING
-DEAL_II_SET_MULTITHREADING_FLAGS
 DEAL_II_CHECK_PARTLY_BRACKETED_INITIALIZER
-AC_SUBST(enablemultithreading)
-
-dnl Next also check whether the MT code shall be used through POSIX
-DEAL_II_CHECK_USE_MT
-AC_SUBST(withmultithreading)
+AC_SUBST(enablethreads)
 
 
 
@@ -192,6 +187,7 @@ DEAL_II_CHECK_SFINAE_BUG
 DEAL_II_CHECK_TEMPL_OP_DISAMBIGUATION_BUG
 DEAL_II_CHECK_ARRAY_CONDITIONAL_DECAY_BUG
 DEAL_II_CHECK_EXPLICIT_CONSTRUCTOR_BUG
+DEAL_II_CHECK_CONST_MEMBER_DEDUCTION_BUG
 DEAL_II_CHECK_TYPE_QUALIFIER_BUG
 DEAL_II_CHECK_WSYNTH_AND_STD_COMPLEX
 DEAL_II_CHECK_CTOR_DTOR_PRIVACY
index 92d11eb998def0f5733455e8fbcd5f9264c59037..8772b62aabea554976fba85f265dad645a826a97 100644 (file)
@@ -7,7 +7,7 @@ include ../common/Make.global_options
 # by default make sublibs. the functionparser is not part of the
 # default make list, since it needs to be built *before* the rest of
 # this stuff as it is needed by libbase
-default: hsl umfpack 
+default: hsl umfpack functionparser tbb
 
 
 ifeq ($(USE_CONTRIB_HSL),yes)
@@ -38,8 +38,19 @@ functionparser:
 endif
 
 
+ifeq ($(enable-threads),yes)
+tbb:
+       @cd tbb ; $(MAKE)
+
+tbb-clean:
+       @cd tbb ; $(MAKE) clean
+else
+tbb tbb-clean:
+endif
+
+
 # generic targets
-clean: hsl-clean
-       @-rm -f ../lib/contrib/*/*.o    
+clean: hsl-clean tbb-clean
+       @-rm -f ../lib/contrib/*/*.o
 
-.PHONY: default hsl clean umfpack functionparser
+.PHONY: default hsl clean hsl-clean umfpack tbb tbb-clean functionparser
index c289e91a7852c9f62cda6439f48544dc188ce176..8b83d5d999b9dc5bd46d1b890df24f8156b9ab12 100644 (file)
@@ -1,19 +1,11 @@
 # $Id$
-# Copyright (C) 2005 by Wolfgang Bangerth
+# Copyright (C) 2005, 2008 by Wolfgang Bangerth
 
 
 D = ../..
 
 include $D/common/Make.global_options
 
-ifneq ($(with-multithreading),no)
-  MT = MT
-else
-  MT = ==
-endif
-
-
-
 # rules
 $(LIBDIR)/contrib/functionparser/fparser.$(OBJEXT): fparser.cc fparser.h
        @echo =====function parser==optimized==$(MT)== $(<F)
index ee5847bf51d39b7c03d4861fc53253a7be8daea6..c297557b9055738e3f15f2c357d9ea723586bcf0 100644 (file)
@@ -49,12 +49,6 @@ go-files = $(go-files-1d) $(go-files-2d) $(go-files-3d)
 # rules how to generate object files from source files. note that
 # there are some files which needs exceptional rules; these are
 # listed immediately below
-ifneq ($(with-multithreading),no)
-  MT = MT
-else
-  MT = ==
-endif
-
 $(LIBDIR)/1d/%.g.$(OBJEXT) :
        @echo "=====deal.II====1d====debug======$(MT)== $(<:source/%=%)"
        @$(CXX) $(CXXFLAGS.g) -Ddeal_II_dimension=1 -c $< -o $@
index a557b1e646f1ad42cadf69f9a002a8360914ef3c..26618816a38cf0acfb61dc9c4b21d356d05a2b3a 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -2077,7 +2077,7 @@ namespace internal
        void
        get_dof_values (const DoFCellAccessor<DoFHandler<dim,spacedim> > &accessor,
                        const InputVector &values,
-                       Vector<number>    &local_values)
+                       dealii::Vector<number> &local_values)
          {
            typedef
              dealii::DoFAccessor<dim,DoFHandler<dim,spacedim> >
@@ -2114,7 +2114,7 @@ namespace internal
        void
        get_dof_values (const DoFCellAccessor<dealii::hp::DoFHandler<dim,spacedim> > &accessor,
                        const InputVector &values,
-                       Vector<number>    &local_values)
+                       dealii::Vector<number>    &local_values)
          {
                                             // no caching for hp::DoFHandler
                                             // implemented
@@ -2137,7 +2137,7 @@ namespace internal
        static
        void
        set_dof_values (const DoFCellAccessor<DoFHandler<dim,spacedim> > &accessor,
-                       const Vector<number> &local_values,
+                       const dealii::Vector<number> &local_values,
                        OutputVector &values)
          {
            typedef
@@ -2167,7 +2167,7 @@ namespace internal
        static
        void
        set_dof_values (const DoFCellAccessor<dealii::hp::DoFHandler<dim,spacedim> > &accessor,
-                       const Vector<number> &local_values,
+                       const dealii::Vector<number> &local_values,
                        OutputVector &values)
          {
                                             // no caching for hp::DoFHandler
index d13a3e6b19fd304a6bf63d17e96ffad97bece3c4..effe6f6dd173076bed4f39a5a43340a9fbc8e1f2 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 by the deal.II authors
+//    Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -406,9 +406,28 @@ class FilteredIterator : public BaseIterator
                                      * Constructor. Use the given
                                      * predicate for filtering and
                                      * initialize the iterator with
-                                     * the given value. This initial
-                                     * value has to satisfy the
-                                     * predicate.
+                                     * the given value.
+                                     *
+                                     * If the initial value @p bi does
+                                     * not satisfy the predicate @p p
+                                     * then it is advanced until we
+                                     * either hit the the
+                                     * past-the-end iterator, or the
+                                     * predicate is satisfied. This
+                                     * allows, for example, to write
+                                     * code like
+                                     * @code
+                                     *   FilteredIterator<typename Triangulation<dim>::active_cell_iterator>
+                                     *     cell (IteratorFilters::SubdomainEqualTo(13),
+                                     *           triangulation.begin_active());
+                                     * @endcode
+                                     *
+                                     * If the cell
+                                     * <code>triangulation.begin_active()</code>
+                                     * does not have a subdomain_id
+                                     * equal to 13, then the iterator
+                                     * will automatically be advanced
+                                     * to the first cell that has.
                                      */
     template <typename Predicate>    
     FilteredIterator (Predicate           p,
@@ -595,7 +614,7 @@ class FilteredIterator : public BaseIterator
                    BaseIterator,
                    << "The element " << arg1
                    << " with which you want to compare or which you want to"
-                   << " assign is invalid since it does not satisfy the predicate.");
+                   << " assign from is invalid since it does not satisfy the predicate.");
     
   private:
 
@@ -712,6 +731,28 @@ class FilteredIterator : public BaseIterator
 
 
 
+/**
+ * Create an object of type FilteredIterator given the base iterator
+ * and predicate.  This function makes the creation of temporary
+ * objects (for example as function arguments) a lot simpler because
+ * one does not have to explicitly specify the type of the base
+ * iterator by hand -- it is deduced automatically here.
+ *
+ * @author Wolfgang Bangerth
+ * @relates FilteredIterator
+ */
+template <typename BaseIterator, typename Predicate>
+FilteredIterator<BaseIterator>
+make_filtered_iterator (const BaseIterator &i,
+                       const Predicate    &p)
+{
+  FilteredIterator<BaseIterator> fi(p);
+  fi.set_to_next_positive (i);
+  return fi;
+}
+
+
+
 /* ------------------ Inline functions and templates ------------ */
 
 
@@ -736,8 +777,9 @@ FilteredIterator (Predicate          p,
                BaseIterator (bi),
                predicate (new PredicateTemplate<Predicate>(p))
 {
-  Assert ((this->state() != IteratorState::valid) || (*predicate) (*this),
-         ExcInvalidElement(bi));
+  if ((this->state() == IteratorState::valid) &&
+      ! (*predicate) (*this))
+    set_to_next_positive (bi);
 }
 
 
index d4cc487e7002b665f3787756b9dabb85239ca6c3..0e91c643f733029ca48c2cc05e80127749850c35 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2003, 2004, 2006, 2007 by the deal.II authors
+//    Copyright (C) 2003, 2004, 2006, 2007, 2008 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -81,7 +81,15 @@ namespace internal
       FEValuesBase (const dealii::hp::FECollection<dim,FEValues::space_dimension> &fe_collection,
                       const dealii::hp::QCollection<q_dim> &q_collection,
                       const UpdateFlags         update_flags);
-        
+
+                                        /**
+                                         * Get a reference to the
+                                         * collection of finite
+                                         * element objects used here.
+                                         */
+       const dealii::hp::FECollection<dim,FEValues::space_dimension> &
+       get_fe_collection () const;
+       
                                          /**
                                           * Return a reference to the @p
                                           * FEValues object selected by the
@@ -1088,6 +1096,16 @@ namespace internal
     {
       return *fe_values_table(present_fe_values_index);
     }
+
+
+
+    template <int dim, int q_dim, class FEValues>
+    inline
+    const dealii::hp::FECollection<dim,FEValues::space_dimension> &
+    FEValuesBase<dim,q_dim,FEValues>::get_fe_collection () const
+    {
+      return *fe_collection;
+    }
   }
   
 }
index fbff012754befa47a30e3d443b3c24559a9775f9..cf4ce13da27570236a61cf780674d4b928ce9fb7 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 
 #include <base/config.h>
 #include <base/smartpointer.h>
-#include <base/multithread_info.h>
 #include <base/data_out_base.h>
 #include <dofs/dof_handler.h>
 #include <fe/mapping.h>
+#include <hp/q_collection.h>
+#include <hp/fe_collection.h>
+#include <hp/mapping_collection.h>
+#include <hp/fe_values.h>
 #include <numerics/data_postprocessor.h>
 #include <numerics/data_component_interpretation.h>
 
@@ -42,45 +45,85 @@ namespace internal
                                      * structures are created
                                      * globally rather than on each
                                      * cell to avoid allocation of
-                                     * memory in the threads.
+                                     * memory in the threads. This is
+                                     * a base class for the
+                                     * AdditionalData kind of data
+                                     * structure discussed in the
+                                     * documentation of the
+                                     * WorkStream class.
                                      *
-                                     * The #cell_to_patch_index_map is
-                                     * an array that stores for index
+                                     * The
+                                     * <code>cell_to_patch_index_map</code>
+                                     * is an array that stores for index
                                      * <tt>[i][j]</tt> the number of the
-                                     * patch that associated with the
-                                     * cell with index @p j on level
-                                     * @p i. This information is set
-                                     * up prior to generation of the
-                                     * patches, and is needed to
-                                     * generate neighborship
+                                     * patch that associated with the cell
+                                     * with index @p j on level @p i. This
+                                     * information is set up prior to
+                                     * generation of the patches, and is
+                                     * needed to generate neighborship
                                      * information.
                                      *
                                      * This structure is used by
                                      * several of the DataOut*
-                                     * classes, not all of which use
-                                     * all fields.
+                                     * classes, which derived their
+                                     * own ParallelData classes from
+                                     * it for additional fields.
                                      */
     template <int dim, int spacedim>
-    struct ParallelData
+    struct ParallelDataBase
     {
-       unsigned int n_threads;
-       unsigned int this_thread;
-       unsigned int n_components;
-       unsigned int n_datasets;
-       unsigned int n_subdivisions;
-       unsigned int n_patches_per_circle;
-       SmartPointer<const Mapping<dim,spacedim> >     mapping;
-       std::vector<double>                            patch_values;
-       std::vector<Vector<double> >                   patch_values_system;
-       std::vector<Tensor<1,spacedim> >               patch_gradients;
-       std::vector<std::vector<Tensor<1,spacedim> > > patch_gradients_system;
-       std::vector<Tensor<2,spacedim> >               patch_hessians;
-       std::vector<std::vector<Tensor<2,spacedim> > > patch_hessians_system;
-       std::vector<Point<spacedim> >                  dummy_normals;
-       std::vector<Point<dim> >                       patch_normals;
-       std::vector<std::vector<Vector<double> > >     postprocessed_values;
-
-       std::vector<std::vector<unsigned int> > *cell_to_patch_index_map;
+       template <class FE>
+       ParallelDataBase (const unsigned int n_components,
+                         const unsigned int n_datasets,
+                         const unsigned int n_subdivisions,
+                         const unsigned int n_q_points,
+                         const std::vector<unsigned int> &n_postprocessor_outputs,
+                         const FE &finite_elements);
+                         
+       const unsigned int n_components;
+       const unsigned int n_datasets;
+       const unsigned int n_subdivisions;
+
+       std::vector<double>                                patch_values;
+       std::vector<dealii::Vector<double> >               patch_values_system;
+       std::vector<Tensor<1,spacedim> >                   patch_gradients;
+       std::vector<std::vector<Tensor<1,spacedim> > >     patch_gradients_system;
+       std::vector<Tensor<2,spacedim> >                   patch_hessians;
+       std::vector<std::vector<Tensor<2,spacedim> > >     patch_hessians_system;
+       std::vector<std::vector<dealii::Vector<double> > > postprocessed_values;
+
+       const dealii::hp::FECollection<dim,spacedim>      fe_collection;        
+    };
+
+
+                                    /**
+                                     * A derived class for use in the
+                                     * DataOut class. This is
+                                     * a class for the
+                                     * AdditionalData kind of data
+                                     * structure discussed in the
+                                     * documentation of the
+                                     * WorkStream context.
+                                     */
+    template <int dim, int spacedim>
+    struct ParallelData : public ParallelDataBase<dim,spacedim>
+    {
+       template <class FE>
+       ParallelData (const Quadrature<dim> &quadrature,
+                     const unsigned int n_components,
+                     const unsigned int n_datasets,
+                     const unsigned int n_subdivisions,
+                     const std::vector<unsigned int> &n_postprocessor_outputs,
+                     const Mapping<dim,spacedim> &mapping,
+                     const std::vector<std::vector<unsigned int> > &cell_to_patch_index_map,
+                     const FE &finite_elements,
+                     const UpdateFlags update_flags);
+
+       const dealii::hp::QCollection<dim> q_collection;
+       const dealii::hp::MappingCollection<dim,spacedim> mapping_collection;
+       dealii::hp::FEValues<dim,spacedim> x_fe_values;
+
+       const std::vector<std::vector<unsigned int> > *cell_to_patch_index_map;
     };
   }
 }
@@ -1199,17 +1242,8 @@ class DataOut : public DataOut_DoFData<DH, DH::dimension, DH::space_dimension>
                                      * stored in
                                      * DataOutInterface::default_subdivisions
                                      * is to be used.
-                                     *
-                                     * The function supports
-                                     * multithreading, if deal.II is
-                                     * compiled in multithreading
-                                     * mode. The default number of
-                                     * threads to be used to build
-                                     * the patches is set to
-                                     * <tt>multithread_info.n_default_threads</tt>.
                                      */
-    virtual void build_patches (const unsigned int n_subdivisions = 0,
-                               const unsigned int n_threads      = multithread_info.n_default_threads);
+    virtual void build_patches (const unsigned int n_subdivisions = 0);
 
                                     /**
                                      * Same as above, except that the
@@ -1261,9 +1295,8 @@ class DataOut : public DataOut_DoFData<DH, DH::dimension, DH::space_dimension>
                                      * replaced by a hp::MappingCollection in
                                      * case of a hp::DoFHandler.
                                      */
-  virtual void build_patches (const Mapping<DH::dimension,DH::space_dimension> &mapping,
+    virtual void build_patches (const Mapping<DH::dimension,DH::space_dimension> &mapping,
                                const unsigned int n_subdivisions = 0,
-                               const unsigned int n_threads      = multithread_info.n_default_threads,
                                const CurvedCellRegion curved_region = curved_boundary);
     
                                     /**
@@ -1309,26 +1342,64 @@ class DataOut : public DataOut_DoFData<DH, DH::dimension, DH::space_dimension>
     
   private:
                                     /**
-                                     * Builds every @p n_threads's
-                                     * patch. This function may be
-                                     * called in parallel.
-                                     * If multithreading is not
-                                     * used, the function is called
-                                     * once and generates all patches.
-                                     */
-    void build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::space_dimension> &data);
-
-                                    /**
-                                     * Store in which region cells shall be
-                                     * curved, if a Mapping is provided.
+                                     * Build one patch. This function
+                                     * is called in a WorkStream
+                                     * context.
                                      */
-    CurvedCellRegion curved_cell_region;
+    void build_one_patch (const std::pair<cell_iterator, unsigned int> *cell_and_index,
+                         internal::DataOut::ParallelData<DH::dimension, DH::space_dimension> &data,
+                         DataOutBase::Patch<DH::dimension, DH::space_dimension> &patch,
+                         const CurvedCellRegion curved_cell_region);
 };
 
 
 
 // -------------------- template and inline functions ------------------------
 
+
+namespace internal
+{
+  namespace DataOut
+  {
+    template <int dim, int spacedim>
+    template <class FE>
+    ParallelDataBase<dim,spacedim>::
+    ParallelDataBase (const unsigned int n_components,
+                     const unsigned int n_datasets,
+                     const unsigned int n_subdivisions,
+                     const unsigned int n_q_points,
+                     const std::vector<unsigned int> &n_postprocessor_outputs,
+                     const FE &finite_elements)
+                   :
+                   n_components (n_components),
+                   n_datasets (n_datasets),
+                   n_subdivisions (n_subdivisions),
+                   patch_values (n_q_points),
+                   patch_values_system (n_q_points),
+                   patch_gradients (n_q_points),
+                   patch_gradients_system (n_q_points),
+                   patch_hessians (n_q_points),
+                   patch_hessians_system (n_q_points),
+                   postprocessed_values (n_postprocessor_outputs.size()),
+                   fe_collection (finite_elements)
+    {
+      for (unsigned int k=0; k<n_q_points; ++k)
+       {
+         patch_values_system[k].reinit(n_components);
+         patch_gradients_system[k].resize(n_components);
+         patch_hessians_system[k].resize(n_components);
+       }
+
+      for (unsigned int dataset=0; dataset<n_postprocessor_outputs.size(); ++dataset)
+       if (n_postprocessor_outputs[dataset] != 0)
+         postprocessed_values[dataset]
+           .resize(n_q_points,
+                   dealii::Vector<double>(n_postprocessor_outputs[dataset]));
+    }
+  }
+}
+
+
 template <class DH, int patch_dim, int patch_space_dim>
 template <class DH2>
 void
index a7ea4180327cbba3e1fffe7815ce3765c2edf4d7..c889a67ed7c0540e1a373ac75922d2819c060150 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 DEAL_II_NAMESPACE_OPEN
 
 
+namespace internal
+{
+  namespace DataOutFaces
+  {
+                                    /**
+                                     * A derived class for use in the
+                                     * DataOutFaces class. This is
+                                     * a class for the
+                                     * AdditionalData kind of data
+                                     * structure discussed in the
+                                     * documentation of the
+                                     * WorkStream context.
+                                     */
+    template <int dim, int spacedim>
+    struct ParallelData : public internal::DataOut::ParallelDataBase<dim,spacedim>
+    {
+       template <class FE>
+       ParallelData (const Quadrature<dim-1> &quadrature,
+                     const unsigned int n_components,
+                     const unsigned int n_datasets,
+                     const unsigned int n_subdivisions,
+                     const std::vector<unsigned int> &n_postprocessor_outputs,
+                     const FE &finite_elements,
+                     const UpdateFlags update_flags);
+
+       const dealii::hp::QCollection<dim-1> q_collection;
+       dealii::hp::FEFaceValues<dim> x_fe_values;
+
+       std::vector<Point<dim> > patch_normals;
+    };
+  }
+}
+
+
 /**
  * This class generates output from faces of a triangulation rather
  * than from cells, as do for example the DataOut and
@@ -110,8 +144,7 @@ class DataOutFaces : public DataOut_DoFData<DH,DH::dimension-1,
                                      * <tt>multithread_info.n_default_threads</tt>.
                                      */
     virtual void
-    build_patches (const unsigned int n_subdivisions = 0,
-                  const unsigned int n_threads      = multithread_info.n_default_threads);
+    build_patches (const unsigned int n_subdivisions = 0);
 
                                     /**
                                      * Declare a way to describe a
@@ -182,14 +215,13 @@ class DataOutFaces : public DataOut_DoFData<DH,DH::dimension-1,
     
   private:
                                     /**
-                                     * Builds every @p n_threads's
-                                     * patch. This function may be
-                                     * called in parallel.
-                                     * If multithreading is not
-                                     * used, the function is called
-                                     * once and generates all patches.
+                                     * Build one patch. This function
+                                     * is called in a WorkStream
+                                     * context.
                                      */
-    void build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &data);
+    void build_one_patch (const FaceDescriptor *cell_and_face,
+                         internal::DataOutFaces::ParallelData<DH::dimension, DH::dimension> &data,
+                         DataOutBase::Patch<DH::dimension-1,DH::space_dimension> &patch);
 };
 
 
index 65b1565a927c64a7dfe576fac1d59a1c5701e79a..af8d501a9e573c9687ee2cc700c79a788c4905a9 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 DEAL_II_NAMESPACE_OPEN
 
 
+namespace internal
+{
+  namespace DataOutRotation
+  {
+                                    /**
+                                     * A derived class for use in the
+                                     * DataOutFaces class. This is
+                                     * a class for the
+                                     * AdditionalData kind of data
+                                     * structure discussed in the
+                                     * documentation of the
+                                     * WorkStream class.
+                                     */
+    template <int dim, int spacedim>
+    struct ParallelData : public internal::DataOut::ParallelDataBase<dim,spacedim>
+    {
+       template <class FE>
+       ParallelData (const Quadrature<dim> &quadrature,
+                     const unsigned int n_components,
+                     const unsigned int n_datasets,
+                     const unsigned int n_subdivisions,
+                     const unsigned int n_patches_per_circle,
+                     const std::vector<unsigned int> &n_postprocessor_outputs,
+                     const FE &finite_elements,
+                     const UpdateFlags update_flags);
+
+       const unsigned int n_patches_per_circle;
+       
+       const dealii::hp::QCollection<dim> q_collection;
+       dealii::hp::FEValues<dim,spacedim> x_fe_values;
+    };
+  }
+}
+
+
+
 /**
  * This class generates output in the full domain of computations that
  * were done using rotational symmetry of domain and solution. In
@@ -119,8 +155,7 @@ class DataOutRotation : public DataOut_DoFData<DH,DH::dimension+1>
                                      * <tt>multithread_info.n_default_threads</tt>.
                                      */
     virtual void build_patches (const unsigned int n_patches_per_circle,
-                               const unsigned int n_subdivisions = 0,
-                               const unsigned int n_threads      = multithread_info.n_default_threads);
+                               const unsigned int n_subdivisions = 0);
 
                                     /**
                                      * Return the first cell which we
@@ -177,7 +212,10 @@ class DataOutRotation : public DataOut_DoFData<DH,DH::dimension+1>
                                      * used, the function is called
                                      * once and generates all patches.
                                      */
-    void build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &data);
+    void
+    build_one_patch (const cell_iterator *cell,
+                    internal::DataOutRotation::ParallelData<DH::dimension, DH::space_dimension> &data,
+                    std::vector<DataOutBase::Patch<DH::dimension+1,DH::space_dimension+1> > &patches);
 };
 
 
index 71c0333daedff26044ab0e8d13799c37f5469a0f..46ae47d84fcc45e7cafb8e3d56bce8859b8d8735 100644 (file)
@@ -32,9 +32,6 @@ namespace hp
 {
   template <int, int> class DoFHandler;
   template <int> class QCollection;
-  template <int, int> class MappingCollection;
-  template <int dim> class FEFaceValues;
-  template <int dim> class FESubfaceValues;
 }
 
 
@@ -256,19 +253,6 @@ class KellyErrorEstimator
                                      * entries, or an empty
                                      * bit-vector.
                                      *
-                                     * The estimator supports
-                                     * multithreading and splits the
-                                     * cells to
-                                     * <tt>multithread_info.n_default_threads</tt>
-                                     * (default) threads. The number
-                                     * of threads to be used in
-                                     * multithreaded mode can be set
-                                     * with the last parameter of the
-                                     * error estimator.
-                                     * Multithreading is only
-                                     * implemented in two and three
-                                     * dimensions.
-                                     *
                                      * The @p subdomain_id parameter
                                      * indicates whether we shall compute
                                      * indicators for all cells (in case its
@@ -307,6 +291,16 @@ class KellyErrorEstimator
                                      * indicators will only be computed for
                                      * cells with this particular material
                                      * id.
+                                     *
+                                     * The @p n_threads parameter used to
+                                     * indicate the number of threads to be
+                                     * used to compute the error
+                                     * estimator. This parameter is now
+                                     * ignored, with the number of threads
+                                     * determined automatically. The
+                                     * parameter is retained for
+                                     * compatibility with old versions of the
+                                     * library.
                                      */
     template <typename InputVector, class DH>
     static void estimate (const Mapping<dim, spacedim>      &mapping,
@@ -505,280 +499,6 @@ class KellyErrorEstimator
                                      * Exception
                                      */
     DeclException0 (ExcNoSolutions);
-
-  private:
-
-
-                                    /**
-                                     * Declare a data type to
-                                     * represent the mapping between
-                                     * faces and integrated jumps of
-                                     * gradients of each of the
-                                     * solution vectors. See the
-                                     * general documentation of this
-                                     * class for more information.
-                                     */
-    template <class DH>
-    struct FaceIntegrals 
-    {
-      typedef
-      std::map<typename DH::face_iterator,std::vector<double> >
-      type;
-    };
-    
-
-                                    /**
-                                     * All small temporary data
-                                     * objects that are needed once
-                                     * per thread by the several
-                                     * functions of the error
-                                     * estimator are gathered in this
-                                     * struct. The reason for this
-                                     * structure is mainly that we
-                                     * have a number of functions
-                                     * that operate on cells or faces
-                                     * and need a number of small
-                                     * temporary data objects. Since
-                                     * these functions may run in
-                                     * parallel, we cannot make these
-                                     * objects member variables of
-                                     * the enclosing class. On the
-                                     * other hand, declaring them
-                                     * locally in each of these
-                                     * functions would require their
-                                     * reallocating every time we
-                                     * visit the next cell or face,
-                                     * which we found can take a
-                                     * significant amount of time if
-                                     * it happens often even in the
-                                     * single threaded case (10-20
-                                     * per cent in our measurements);
-                                     * however, most importantly,
-                                     * memory allocation requires
-                                     * synchronisation in
-                                     * multithreaded mode. While that
-                                     * is done by the C++ library and
-                                     * has not to be handcoded, it
-                                     * nevertheless seriously damages
-                                     * the ability to efficiently run
-                                     * the functions of this class in
-                                     * parallel, since they are quite
-                                     * often blocked by these
-                                     * synchronisation points,
-                                     * slowing everything down by a
-                                     * factor of two or three.
-                                     *
-                                     * Thus, every thread gets an
-                                     * instance of this class to work
-                                     * with and needs not allocate
-                                     * memory itself, or synchronise
-                                     * with other threads.
-                                     *
-                                     * The sizes of the arrays are
-                                     * initialized with the maximal number of
-                                     * entries necessary for the hp
-                                     * case. Within the loop over individual
-                                     * cells, we then resize the arrays as
-                                     * necessary. Since for std::vector
-                                     * resizing to a smaller size doesn't
-                                     * imply memory allocation, this is fast.
-                                     */
-    struct PerThreadData
-    {
-                                        /**
-                                         * A vector to store the jump
-                                         * of the normal vectors in
-                                         * the quadrature points for
-                                         * each of the solution
-                                         * vectors (i.e. a temporary
-                                         * value). This vector is not
-                                         * allocated inside the
-                                         * functions that use it, but
-                                         * rather globally, since
-                                         * memory allocation is slow,
-                                         * in particular in presence
-                                         * of multiple threads where
-                                         * synchronisation makes
-                                         * things even slower.
-                                         */ 
-       std::vector<std::vector<std::vector<double> > >         phi;
-
-                                        /**
-                                         * A vector for the gradients of
-                                         * the finite element function
-                                         * on one cell
-                                         *
-                                         * Let psi be a short name
-                                         * for <tt>a grad u_h</tt>, where
-                                         * the third index be the
-                                         * component of the finite
-                                         * element, and the second
-                                         * index the number of the
-                                         * quadrature point. The
-                                         * first index denotes the
-                                         * index of the solution
-                                         * vector.
-                                         */
-       std::vector<std::vector<std::vector<Tensor<1,dim> > > > psi;
-
-                                        /**
-                                         * The same vector for a neighbor cell
-                                         */
-       std::vector<std::vector<std::vector<Tensor<1,dim> > > > neighbor_psi;
-
-                                        /**
-                                         * The normal vectors of the finite
-                                         * element function on one face
-                                         */
-       std::vector<Point<dim> > normal_vectors;
-
-                                        /**
-                                         * Two arrays needed for the
-                                         * values of coefficients in
-                                         * the jumps, if they are
-                                         * given.
-                                         */
-       std::vector<double>          coefficient_values1;
-       std::vector<Vector<double> > coefficient_values;
-
-                                        /**
-                                         * Array for the products of
-                                         * Jacobian determinants and
-                                         * weights of quadraturs
-                                         * points.
-                                         */
-       std::vector<double>          JxW_values;
-
-                                         /**
-                                          * The subdomain id we are to care
-                                          * for.
-                                          */
-        const unsigned int subdomain_id;
-                                         /**
-                                          * The material id we are to care
-                                          * for.
-                                          */
-        const unsigned int material_id;
-        
-                                        /**
-                                         * Constructor.
-                                         */
-       PerThreadData (const unsigned int n_solution_vectors,
-                      const unsigned int n_components,
-                      const unsigned int max_n_q_points,
-                       const unsigned int subdomain_id,
-                       const unsigned int material_id);
-
-                                        /**
-                                         * Constructor.
-                                         */
-       void resize (const unsigned int n_components,
-                    const unsigned int max_n_q_points);
-    };
-
-
-                                    /**
-                                     * Computates the error on all cells
-                                     * of the domain with the number n,
-                                     * satisfying
-                                     * <tt>n=this_thread (mod n_threads)</tt>
-                                     * This enumeration is chosen to
-                                     * generate a random distribution
-                                     * of all cells.
-                                     *
-                                     * This function is only needed
-                                     * in two or three dimensions.
-                                     * The error estimator in one
-                                     * dimension is implemented
-                                     * seperatly.
-                                     */
-    template <typename InputVector, class DH>
-    static void estimate_some (const hp::MappingCollection<dim, spacedim>    &mapping,
-                               const DH                            &dof_handler,
-                               const hp::QCollection<dim-1>         &quadrature,
-                               const typename FunctionMap<dim>::type &neumann_bc,
-                               const std::vector<const InputVector *> &solutions,
-                              const std::pair<const std::vector<bool>*,const Function<dim>*> &component_mask_and_coefficients,
-                               const std::pair<unsigned int, unsigned int> this_thread,
-                               typename FaceIntegrals<DH>::type    &face_integrals,
-                               PerThreadData                       &per_thread_data);
-                               
-                                    /**
-                                     * Actually do the computation on
-                                     * a face which has no hanging
-                                     * nodes (it is regular), i.e.
-                                     * either on the other side there
-                                     * is nirvana (face is at
-                                     * boundary), or the other side's
-                                     * refinement level is the same
-                                     * as that of this side, then
-                                     * handle the integration of
-                                     * these both cases together.
-                                     *
-                                     * The meaning of the parameters
-                                     * becomes clear when looking at
-                                     * the source code. This function
-                                     * is only externalized from
-                                     * @p estimate_error to avoid
-                                     * ending up with a function of
-                                     * 500 lines of code.
-                                     */
-    template <typename InputVector, class DH>
-    static
-    void
-    integrate_over_regular_face (const DH                             &dof_handler,
-                                 const hp::QCollection<dim-1>         &quadrature,
-                                 const typename FunctionMap<dim>::type &neumann_bc,
-                                 const std::vector<const InputVector *> &solutions,
-                                 const std::vector<bool>                  &component_mask,
-                                 const Function<dim>                 *coefficients,
-                                 typename FaceIntegrals<DH>::type    &face_integrals,
-                                 PerThreadData              &per_thread_data,
-                                 const typename DH::active_cell_iterator &cell,
-                                 const unsigned int          face_no,
-                                 hp::FEFaceValues<dim>          &fe_face_values_cell,
-                                 hp::FEFaceValues<dim>          &fe_face_values_neighbor);
-
-
-                                    /**
-                                     * The same applies as for the
-                                     * function above, except that
-                                     * integration is over face
-                                     * @p face_no of @p cell, where
-                                     * the respective neighbor is
-                                     * refined, so that the
-                                     * integration is a bit more
-                                     * complex.
-                                     */
-    template <typename InputVector, class DH>
-    static
-    void
-    integrate_over_irregular_face (const DH                             &dof_handler,
-                                   const hp::QCollection<dim-1>         &quadrature,
-                                   const std::vector<const InputVector *> &solutions,
-                                   const std::vector<bool>                  &component_mask,
-                                   const Function<dim>                 *coefficients,
-                                   typename FaceIntegrals<DH>::type    &face_integrals,
-                                   PerThreadData              &per_thread_data,
-                                   const typename DH::active_cell_iterator &cell,
-                                   const unsigned int          face_no,
-                                   hp::FEFaceValues<dim>          &fe_face_values,
-                                   hp::FESubfaceValues<dim>       &fe_subface_values);
-
-                                    /** 
-                                     * By the resolution of Defect
-                                     * Report 45 to the ISO C++ 1998
-                                     * standard, nested classes
-                                     * automatically have access to
-                                     * members of the enclosing
-                                     * class. Nevertheless, some
-                                     * compilers don't implement this
-                                     * resolution yet, so we have to
-                                     * make them @p friend, which
-                                     * doesn't hurt on the other
-                                     * compilers as well.
-                                     */
-    friend class PerThreadData;
 };
 
 
index a19accf83894a91b82773f07d1ca305ca3a68272..4374632304c335bfa1bc06aa289bf91a52829c3d 100644 (file)
@@ -629,160 +629,6 @@ class MatrixCreator
        active_cell_iterator first, second;
     };
     
-    
-                                    /**
-                                     * Version of the same function
-                                     * (without suffix @p _1) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, typename number, int spacedim>
-    static
-    void create_mass_matrix_1 (const Mapping<dim, spacedim>       &mapping,
-                              const DoFHandler<dim,spacedim>    &dof,
-                              const Quadrature<dim>    &q,
-                              SparseMatrix<number>     &matrix,
-                              const Function<spacedim> * const a,
-                              const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                              Threads::ThreadMutex     &mutex);
-
-                                    /**
-                                     * Version of the same function
-                                     * (without suffix @p _2) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, typename number, int spacedim>
-    static
-    void create_mass_matrix_2 (const Mapping<dim, spacedim>       &mapping,
-                              const DoFHandler<dim,spacedim>    &dof,
-                              const Quadrature<dim>    &q,
-                              SparseMatrix<number>     &matrix,
-                              const Function<spacedim>      &rhs,
-                              Vector<double>           &rhs_vector,
-                              const Function<spacedim> * const a,
-                              const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                              Threads::ThreadMutex     &mutex);
-
-                                    /**
-                                     * Same function as above, but for hp
-                                     * objects.
-                                     */
-    template <int dim, typename number, int spacedim>
-    static
-    void create_mass_matrix_1 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                              const hp::DoFHandler<dim,spacedim>    &dof,
-                              const hp::QCollection<dim>    &q,
-                              SparseMatrix<number>     &matrix,
-                              const Function<spacedim> * const a,
-                              const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                              Threads::ThreadMutex     &mutex);
-
-    template <int dim, typename number, int spacedim>
-    static
-    void create_mass_matrix_2 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                              const hp::DoFHandler<dim,spacedim>    &dof,
-                              const hp::QCollection<dim>    &q,
-                              SparseMatrix<number>     &matrix,
-                              const Function<spacedim>      &rhs,
-                              Vector<double>           &rhs_vector,
-                              const Function<spacedim> * const a,
-                              const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                              Threads::ThreadMutex     &mutex);
-
-
-/**
-                                     * Version of the same function
-                                     * (without suffix @p _1) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, int spacedim>
-    static
-    void create_laplace_matrix_1 (const Mapping<dim, spacedim>       &mapping,
-                                 const DoFHandler<dim,spacedim>    &dof,
-                                 const Quadrature<dim>    &q,
-                                 SparseMatrix<double>     &matrix,
-                                 const Function<spacedim> * const a,
-                                 const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                 Threads::ThreadMutex     &mutex);
-
-                                    /**
-                                     * Version of the same function
-                                     * (without suffix @p _2) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, int spacedim>
-    static
-    void create_laplace_matrix_2 (const Mapping<dim, spacedim>       &mapping,
-                                 const DoFHandler<dim,spacedim>    &dof,
-                                 const Quadrature<dim>    &q,
-                                 SparseMatrix<double>     &matrix,
-                                 const Function<spacedim>      &rhs,
-                                 Vector<double>           &rhs_vector,
-                                 const Function<spacedim> * const a,
-                                 const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                 Threads::ThreadMutex     &mutex);
-
-                                    /**
-                                     * Version of the same function
-                                     * (without suffix @p _1) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, int spacedim>
-    static
-    void create_laplace_matrix_1 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                 const hp::DoFHandler<dim,spacedim>    &dof,
-                                 const hp::QCollection<dim>    &q,
-                                 SparseMatrix<double>     &matrix,
-                                 const Function<spacedim> * const a,
-                                 const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                 Threads::ThreadMutex     &mutex);
-
-                                    /**
-                                     * Version of the same function
-                                     * (without suffix @p _2) with
-                                     * the same argument list that
-                                     * operates only on an interval
-                                     * of iterators. Used for
-                                     * parallelization. The mutex is
-                                     * used to synchronise access to
-                                     * the matrix.
-                                     */
-    template <int dim, int spacedim>
-    static
-    void create_laplace_matrix_2 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                 const hp::DoFHandler<dim,spacedim>    &dof,
-                                 const hp::QCollection<dim>    &q,
-                                 SparseMatrix<double>     &matrix,
-                                 const Function<spacedim>      &rhs,
-                                 Vector<double>           &rhs_vector,
-                                 const Function<spacedim> * const a,
-                                 const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                 Threads::ThreadMutex     &mutex);
-    
                                     /**
                                      * Version of the same function
                                      * (without suffix @p _1) with
index 1a883eb954956e3ab5404c6388f633231e9c0f6e..b934d2f490ae4948b3e85970a6c2e0296445bdfe 100644 (file)
@@ -3507,18 +3507,18 @@ namespace internal
 
       const unsigned int max_n_q_points = q.max_n_quadrature_points ();
       
-      std::vector< Vector<double> >
-        function_values (max_n_q_points, Vector<double>(n_components));
+      std::vector< dealii::Vector<double> >
+        function_values (max_n_q_points, dealii::Vector<double>(n_components));
       std::vector<std::vector<Tensor<1,spacedim> > >
         function_grads (max_n_q_points, std::vector<Tensor<1,spacedim> >(n_components));
 
       std::vector<double>
         weight_values (max_n_q_points);
-      std::vector<Vector<double> >
-        weight_vectors (max_n_q_points, Vector<double>(n_components));
+      std::vector<dealii::Vector<double> >
+        weight_vectors (max_n_q_points, dealii::Vector<double>(n_components));
 
-      std::vector<Vector<double> >
-        psi_values (max_n_q_points, Vector<double>(n_components));
+      std::vector<dealii::Vector<double> >
+        psi_values (max_n_q_points, dealii::Vector<double>(n_components));
       std::vector<std::vector<Tensor<1,spacedim> > >
         psi_grads (max_n_q_points, std::vector<Tensor<1,spacedim> >(n_components));
       std::vector<double>
@@ -3547,16 +3547,16 @@ namespace internal
                                            // quadrature points we use
                                            // for the present cell
           function_values.resize (n_q_points,
-                                  Vector<double>(n_components));
+                                  dealii::Vector<double>(n_components));
           function_grads.resize (n_q_points,
                                  std::vector<Tensor<1,spacedim> >(n_components));
 
           weight_values.resize (n_q_points);
           weight_vectors.resize (n_q_points,
-                                 Vector<double>(n_components));
+                                 dealii::Vector<double>(n_components));
 
           psi_values.resize (n_q_points,
-                             Vector<double>(n_components));
+                             dealii::Vector<double>(n_components));
           psi_grads.resize (n_q_points,
                             std::vector<Tensor<1,spacedim> >(n_components));
           psi_scalar.resize (n_q_points);
index 67d3c395267811e3169fc0cfaf42f4c92ae3cddc..a79a813d002ce90b8ec55695fda4d1068d0a46f8 100644 (file)
@@ -563,7 +563,7 @@ namespace DoFRenumbering
   
     Assert (result == dof_handler.n_dofs(level),
            ExcRenumberingIncomplete());
-  
+
     if (renumbering.size()!=0)
       dof_handler.renumber_dofs (level, renumbering);
   }
@@ -575,19 +575,23 @@ namespace DoFRenumbering
   component_wise (MGDoFHandler<dim> &dof_handler,
                  const std::vector<unsigned int> &component_order_arg)
   {
-    Threads::ThreadGroup<> threads;
-
+                                    // renumber the non-MG part of
+                                    // the DoFHandler in parallel to
+                                    // the MG part. Because
+                                    // MGDoFHandler::renumber_dofs
+                                    // uses the user flags we can't
+                                    // run renumbering on individual
+                                    // levels in parallel to the
+                                    // other levels
     void (*non_mg_part) (DoFHandler<dim> &, const std::vector<unsigned int> &)
       = &component_wise<dim>;
-    void (*mg_part) (MGDoFHandler<dim> &, unsigned int, const std::vector<unsigned int> &)
-      = &component_wise<dim>;
-  
-    threads += Threads::spawn (non_mg_part) (static_cast<DoFHandler<dim>&> (dof_handler),
-                                            component_order_arg);
+    Threads::Task<>
+      task = Threads::new_task (non_mg_part, dof_handler, component_order_arg);
+    
     for (unsigned int level=0; level<dof_handler.get_tria().n_levels(); ++level)
-      threads += Threads::spawn (mg_part) (dof_handler, level, component_order_arg);
+      component_wise (dof_handler, level, component_order_arg);
 
-    threads.join_all();
+    task.join();
   }
 
 
index d75a185883b7ed10cbb0ada42e2ef938d269c0eb..94108bdbda3c6eea6ddbca8cd03ba042a937d3e9 100644 (file)
@@ -3999,7 +3999,7 @@ DoFTools::count_dofs_per_component (
   std::vector<std::vector<bool> >
     component_select (n_components,
                       std::vector<bool>(n_components, false));
-  Threads::ThreadGroup<> threads;
+  Threads::TaskGroup<> tasks;
   for (unsigned int i=0; i<n_components; ++i)
     {
       void (*fun_ptr) (const DoFHandler<dim,spacedim>   &,
@@ -4008,10 +4008,11 @@ DoFTools::count_dofs_per_component (
                       bool)
         = &DoFTools::template extract_dofs<dim>;
       component_select[i][i] = true;
-      threads += Threads::spawn (fun_ptr)(dof_handler, component_select[i],
-                                          dofs_in_component[i], false);
-    };
-  threads.join_all ();
+      tasks += Threads::new_task (fun_ptr,
+                                 dof_handler, component_select[i],
+                                 dofs_in_component[i], false);
+    }
+  tasks.join_all ();
 
                                   // next count what we got
   unsigned int component = 0;
@@ -4099,7 +4100,7 @@ DoFTools::count_dofs_per_block (
     dofs_in_block (n_blocks, std::vector<bool>(dof_handler.n_dofs(), false));
   std::vector<std::vector<bool> >
     block_select (n_blocks, std::vector<bool>(n_blocks, false));
-  Threads::ThreadGroup<> threads;
+  Threads::TaskGroup<> tasks;
   for (unsigned int i=0; i<n_blocks; ++i)
     {
       void (*fun_ptr) (const DoFHandler<dim,spacedim>   &,
@@ -4108,10 +4109,11 @@ DoFTools::count_dofs_per_block (
                       bool)
         = &DoFTools::template extract_dofs<dim>;
       block_select[i][i] = true;
-      threads += Threads::spawn (fun_ptr)(dof_handler, block_select[i],
-                                          dofs_in_block[i], true);
+      tasks += Threads::new_task (fun_ptr,
+                                 dof_handler, block_select[i],
+                                 dofs_in_block[i], true);
     };
-  threads.join_all ();
+  tasks.join_all ();
 
                                   // next count what we got
   for (unsigned int block=0;block<fe.n_blocks();++block)
@@ -4244,8 +4246,9 @@ namespace internal
                                       // degree of freedom of the
                                       // coarse-grid variable in the
                                       // fine-grid element
-      std::vector<Vector<double> > parameter_dofs (coarse_dofs_per_cell_component,
-                                                  Vector<double>(fine_dofs_per_cell));
+      std::vector<dealii::Vector<double> >
+       parameter_dofs (coarse_dofs_per_cell_component,
+                       dealii::Vector<double>(fine_dofs_per_cell));
                                       // for each coarse dof: find its
                                       // position within the fine element
                                       // and set this value to one in the
@@ -4399,7 +4402,7 @@ namespace internal
       const dealii::DoFHandler<dim,spacedim>              &coarse_grid,
       const unsigned int                  coarse_component,
       const InterGridMap<dealii::DoFHandler<dim,spacedim> > &coarse_to_fine_grid_map,
-      const std::vector<Vector<double> > &parameter_dofs,
+      const std::vector<dealii::Vector<double> > &parameter_dofs,
       const std::vector<int>             &weight_mapping,
       std::vector<std::map<unsigned int, float> > &weights,
       const typename dealii::DoFHandler<dim,spacedim>::active_cell_iterator &begin,
@@ -4479,7 +4482,7 @@ namespace internal
                                       // coarse grid (for the selected fe)
                                       // on the fine grid
       const unsigned int n_fine_dofs = weight_mapping.size();
-      Vector<double> global_parameter_representation (n_fine_dofs);
+      dealii::Vector<double> global_parameter_representation (n_fine_dofs);
   
       typename dealii::DoFHandler<dim,spacedim>::active_cell_iterator cell;
       std::vector<unsigned int> parameter_dof_indices (coarse_fe.dofs_per_cell);
@@ -4590,7 +4593,7 @@ namespace internal
       const dealii::DoFHandler<dim,spacedim>              &coarse_grid,
       const unsigned int                  coarse_component,
       const InterGridMap<dealii::DoFHandler<dim,spacedim> > &coarse_to_fine_grid_map,
-      const std::vector<Vector<double> > &parameter_dofs,
+      const std::vector<dealii::Vector<double> > &parameter_dofs,
       const std::vector<int>             &weight_mapping,
       std::vector<std::map<unsigned int,float> > &weights)
     {
@@ -4602,25 +4605,27 @@ namespace internal
                                                                     coarse_grid.end(),
                                                                     multithread_info.n_default_threads);
 
-      Threads::ThreadGroup<> threads;
+//TODO: use WorkStream here      
+      Threads::TaskGroup<> tasks;
       void (*fun_ptr) (const dealii::DoFHandler<dim,spacedim>              &,
                       const unsigned int                  ,
                       const InterGridMap<dealii::DoFHandler<dim,spacedim> > &,
-                      const std::vector<Vector<double> > &,
+                      const std::vector<dealii::Vector<double> > &,
                       const std::vector<int>             &,
                       std::vector<std::map<unsigned int, float> > &,
                       const typename dealii::DoFHandler<dim,spacedim>::active_cell_iterator &,
                       const typename dealii::DoFHandler<dim,spacedim>::active_cell_iterator &)
        = &compute_intergrid_weights_3<dim>;
       for (unsigned int i=0; i<multithread_info.n_default_threads; ++i)
-       threads += Threads::spawn (fun_ptr)(coarse_grid, coarse_component,
-                                           coarse_to_fine_grid_map, parameter_dofs,
-                                           weight_mapping, weights,
-                                           cell_intervals[i].first,
-                                           cell_intervals[i].second);
-
-                                      // wait for the threads to finish
-      threads.join_all ();
+       tasks += Threads::new_task (fun_ptr,
+                                   coarse_grid, coarse_component,
+                                   coarse_to_fine_grid_map, parameter_dofs,
+                                   weight_mapping, weights,
+                                   cell_intervals[i].first,
+                                   cell_intervals[i].second);
+
+                                      // wait for the tasks to finish
+      tasks.join_all ();
     }
   }
 }
index a959825bc3c1dc30e33758cedbce942d14f14f15..3d1135478a778d30c05fadf5e9ca83a67f4223fd 100644 (file)
@@ -2547,10 +2547,11 @@ void GridGenerator::laplace_transformation (Triangulation<dim> &tria,
     us[i].reinit (dof_handler.n_dofs());
   
                                   // solve linear systems in parallel
-  Threads::ThreadGroup<> threads;
+  Threads::TaskGroup<> tasks;
   for (unsigned int i=0; i<dim; ++i)
-    threads += Threads::spawn (&GridGenerator::laplace_solve)(S, m[i], us[i]);
-  threads.join_all ();
+    tasks += Threads::new_task (&GridGenerator::laplace_solve,
+                               S, m[i], us[i]);
+  tasks.join_all ();
   
                                   // change the coordinates of the
                                   // points of the triangulation
index cba09ad05c2bba0af2e4c7687743ac690dbea8c9..8ccc6d752b90daf0f91d2789ab9d9e837b9ac5e8 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -253,12 +253,12 @@ namespace internal
     void
     get_mg_dof_values (const dealii::MGDoFAccessor<1, dim, spacedim> &accessor,
                       const int level,
-                      const Vector<number> &values,
-                      Vector<number>       &dof_values)
+                      const dealii::Vector<number> &values,
+                      dealii::Vector<number>       &dof_values)
     {
       const unsigned int dofs_per_vertex = accessor.get_dof_handler().get_fe().dofs_per_vertex,
                         dofs_per_line   = accessor.get_dof_handler().get_fe().dofs_per_line;
-      typename Vector<number>::iterator next_dof_value=dof_values.begin();
+      typename dealii::Vector<number>::iterator next_dof_value=dof_values.begin();
       for (unsigned int vertex=0; vertex<2; ++vertex)
        for (unsigned int d=0; d<dofs_per_vertex; ++d)
          *next_dof_value++ = values(accessor.mg_vertex_dof_index(level,vertex,d));
@@ -300,13 +300,13 @@ namespace internal
     void
     get_mg_dof_values (const dealii::MGDoFAccessor<2, dim, spacedim> &accessor,
                       const int level,
-                      const Vector<number> &values,
-                      Vector<number>       &dof_values)
+                      const dealii::Vector<number> &values,
+                      dealii::Vector<number>       &dof_values)
     {
       const unsigned int dofs_per_vertex = accessor.get_dof_handler().get_fe().dofs_per_vertex,
                         dofs_per_line   = accessor.get_dof_handler().get_fe().dofs_per_line,
                         dofs_per_quad   = accessor.get_dof_handler().get_fe().dofs_per_quad;
-      typename Vector<number>::iterator next_dof_value=dof_values.begin();
+      typename dealii::Vector<number>::iterator next_dof_value=dof_values.begin();
       for (unsigned int vertex=0; vertex<4; ++vertex)
        for (unsigned int d=0; d<dofs_per_vertex; ++d)
          *next_dof_value++ = values(accessor.mg_vertex_dof_index(level,vertex,d));
@@ -353,14 +353,14 @@ namespace internal
     void
     get_mg_dof_values (const dealii::MGDoFAccessor<3, dim, spacedim> &accessor,
                       const int level,
-                      const Vector<number> &values,
-                      Vector<number>       &dof_values)
+                      const dealii::Vector<number> &values,
+                      dealii::Vector<number>       &dof_values)
     {
       const unsigned int dofs_per_vertex = accessor.get_dof_handler().get_fe().dofs_per_vertex,
                         dofs_per_line   = accessor.get_dof_handler().get_fe().dofs_per_line,
                         dofs_per_quad   = accessor.get_dof_handler().get_fe().dofs_per_quad,
                         dofs_per_hex    = accessor.get_dof_handler().get_fe().dofs_per_hex;
-      typename Vector<number>::iterator next_dof_value=dof_values.begin();
+      typename dealii::Vector<number>::iterator next_dof_value=dof_values.begin();
       for (unsigned int vertex=0; vertex<8; ++vertex)
        for (unsigned int d=0; d<dofs_per_vertex; ++d)
          *next_dof_value++ = values(accessor.mg_vertex_dof_index(level,vertex,d));
index 742e9d4d6de6a1c919daa45ca4d0b39e78fd3804..579875db2c1c8accdcec21bbdbf5fdeb04aca9d0 100644 (file)
@@ -956,7 +956,7 @@ MGTools::count_dofs_per_component (
          std::vector<std::vector<bool> >
            component_select (n_components,
                              std::vector<bool>(n_components, false));
-         Threads::ThreadGroup<> threads;
+         Threads::TaskGroup<> tasks;
          for (unsigned int i=0; i<n_components; ++i)
            {
              void (*fun_ptr) (const unsigned int       level,
@@ -966,10 +966,12 @@ MGTools::count_dofs_per_component (
                               bool)
                = &DoFTools::template extract_level_dofs<dim>;
              component_select[i][i] = true;
-             threads += Threads::spawn (fun_ptr)(l, dof_handler, component_select[i],
-                                                  dofs_in_component[i], false);
+             tasks += Threads::new_task (fun_ptr,
+                                         l, dof_handler,
+                                         component_select[i],
+                                         dofs_in_component[i], false);
            }
-         threads.join_all();
+         tasks.join_all();
 
                                           // next count what we got
          unsigned int component = 0;
@@ -1067,7 +1069,7 @@ MGTools::count_dofs_per_block (
        dofs_in_block (n_blocks, std::vector<bool>(dof_handler.n_dofs(l), false));
       std::vector<std::vector<bool> >
        block_select (n_blocks, std::vector<bool>(n_blocks, false));
-      Threads::ThreadGroup<> threads;
+      Threads::TaskGroup<> tasks;
       for (unsigned int i=0; i<n_blocks; ++i)
        {
          void (*fun_ptr) (const unsigned int level,
@@ -1077,10 +1079,11 @@ MGTools::count_dofs_per_block (
                           bool)
            = &DoFTools::template extract_level_dofs<dim>;
          block_select[i][i] = true;
-         threads += Threads::spawn (fun_ptr)(l, dof_handler, block_select[i],
-                                             dofs_in_block[i], true);
+         tasks += Threads::new_task (fun_ptr,
+                                     l, dof_handler, block_select[i],
+                                     dofs_in_block[i], true);
        };
-      threads.join_all ();
+      tasks.join_all ();
       
                                       // next count what we got
       for (unsigned int block=0;block<fe.n_blocks();++block)
index 353f20cd39a8fdf7d66dbe12eb138e0c3a8f1cec..82a92b75121966684cc7c1ecc2ff3679e6aeb0c0 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -13,8 +13,8 @@
 
 
 #include <base/quadrature_lib.h>
+#include <base/work_stream.h>
 #include <base/memory_consumption.h>
-#include <base/thread_management.h>
 #include <lac/vector.h>
 #include <lac/block_vector.h>
 #include <lac/petsc_vector.h>
 DEAL_II_NAMESPACE_OPEN
 
 
+namespace internal
+{
+  namespace DataOut
+  {
+    template <int dim, int spacedim>
+    template <class FE>
+    ParallelData<dim,spacedim>::
+    ParallelData (const Quadrature<dim> &quadrature,
+                 const unsigned int n_components,
+                 const unsigned int n_datasets,
+                 const unsigned int n_subdivisions,
+                 const std::vector<unsigned int> &n_postprocessor_outputs,
+                 const Mapping<dim,spacedim> &mapping,
+                 const std::vector<std::vector<unsigned int> > &cell_to_patch_index_map,
+                 const FE &finite_elements,
+                 const UpdateFlags update_flags)
+                   :
+                   ParallelDataBase<dim,spacedim> (n_components,
+                                                   n_datasets,
+                                                   n_subdivisions,
+                                                   quadrature.n_quadrature_points,
+                                                   n_postprocessor_outputs,
+                                                   finite_elements),
+                   q_collection (quadrature),
+                   mapping_collection (mapping),           
+                   x_fe_values (this->mapping_collection,
+                                this->fe_collection,
+                                q_collection,
+                                update_flags),
+                   cell_to_patch_index_map (&cell_to_patch_index_map)
+    {}
+
+
+
+
+                                    /**
+                                     * In a WorkStream context, use
+                                     * this function to append the
+                                     * patch computed by the parallel
+                                     * stage to the array of patches.
+                                     */
+    template <int dim, int spacedim>
+    void
+    append_patch_to_list (const DataOutBase::Patch<dim,spacedim> &patch,
+                         std::vector<DataOutBase::Patch<dim,spacedim> > &patches)
+    {
+      patches.push_back (patch);
+      patches.back().patch_index = patches.size()-1;
+    }
+  }
+}
+
+
 template <class DH, int patch_dim, int patch_space_dim>
 DataOut_DoFData<DH,patch_dim,patch_space_dim>::
 DataEntryBase::DataEntryBase (const std::vector<std::string> &names_in,
@@ -618,380 +671,272 @@ DataOut_DoFData<DH,patch_dim,patch_space_dim>::memory_consumption () const
 /* ---------------------------------------------------------------------- */
 
 
-
 template <int dim, class DH>
-void DataOut<dim,DH>::build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::space_dimension> &data)
-{
-                                  // Check consistency of redundant
-                                  // template parameter
-  Assert (dim==DH::dimension, ExcDimensionMismatch(dim, DH::dimension));
-  
-  QTrapez<1>     q_trapez;
-  QIterated<DH::dimension> patch_points (q_trapez, data.n_subdivisions);
-
-// We use the mapping to transform the vertex coordinates and the shape
-// functions (necessary for example for Raviart-Thomas elements). On the
-// boundary, general mappings do not reduce to a MappingQ1, therefore the mapped
-// (quadrature) points are stored in the patch, whereas for cells in the
-// interior of the domain these points are obtained by a dim-linear mapping and
-// can be recovered from the vertices later on, thus they need not to be stored.
-
-                                  // create collection objects from
-                                  // single quadratures,
-                                  // mappings and finite elements. if we have
-                                  // an hp DoFHandler,
-                                  // dof_handler.get_fe() returns a
-                                  // collection of which we do a
-                                  // shallow copy instead
-  const hp::QCollection<DH::dimension>       q_collection (patch_points);
-  const hp::FECollection<DH::dimension,DH::space_dimension>      fe_collection(this->dofs->get_fe());
-  const hp::MappingCollection<DH::dimension,DH::space_dimension> mapping_collection(*(data.mapping));
-
-  UpdateFlags update_flags=update_values;
-  if (curved_cell_region != no_curved_cells)
-    update_flags |= update_quadrature_points;
-  
-  for (unsigned int i=0; i<this->dof_data.size(); ++i)
-    if (this->dof_data[i]->postprocessor)
-      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
-                                  // perhaps update_normal_vectors is present,
-                                  // which would only be useful on faces, but
-                                  // we may not use it here.
-  Assert (!(update_flags & update_normal_vectors),
-         ExcMessage("The update of normal vectors may not be requested for evaluation of data on cells via DataPostprocessor."));
-  
-  hp::FEValues<DH::dimension,DH::space_dimension> x_fe_patch_values (mapping_collection,
-                                                fe_collection,
-                                                q_collection,
-                                                update_flags);
-
-  const unsigned int n_q_points = patch_points.n_quadrature_points;
-  
-  typename std::vector< dealii::DataOutBase::Patch<DH::dimension,DH::space_dimension> >::iterator
-    patch = this->patches.begin();
-  cell_iterator cell=first_cell();
-
-                                  // keep track of the index of the
-                                  // current cell so we can
-                                  // efficiently evaluate cell-based
-                                  // data (as opposed to DoF-based
-                                  // data). we do so only if
-                                  // this->cell_data.size() != 0
-  unsigned int cell_index = (this->cell_data.size() != 0
-                            ?
-                            std::distance (this->dofs->begin_active(),
-                                           active_cell_iterator (cell))
-                            :
-                            numbers::invalid_unsigned_int);
-  
-  
-                                  // get first cell in this thread
-  for (unsigned int i=0; (i<data.this_thread)&&(cell != this->dofs->end()); ++i)
-    {
-      ++patch;
-
-      const cell_iterator old_cell = cell;
+void
+DataOut<dim,DH>::
+build_one_patch (const std::pair<cell_iterator, unsigned int> *cell_and_index,
+                internal::DataOut::ParallelData<DH::dimension, DH::space_dimension> &data,
+                DataOutBase::Patch<DH::dimension, DH::space_dimension> &patch,
+                const CurvedCellRegion curved_cell_region)
+{  
+                                  // use ucd_to_deal map as patch
+                                  // vertices are in the old,
+                                  // unnatural ordering
+  for (unsigned int vertex=0; vertex<GeometryInfo<DH::dimension>::vertices_per_cell; ++vertex)
+    patch.vertices[vertex] = data.mapping_collection[0].transform_unit_to_real_cell
+                            (cell_and_index->first,
+                             GeometryInfo<DH::dimension>::unit_cell_vertex (vertex));
       
-      cell = next_cell(cell);
-
-      if (this->cell_data.size() != 0)
-       cell_index += std::distance (active_cell_iterator(old_cell),
-                                    active_cell_iterator(cell));
-    }
-
-                                  // now loop over all cells and
-                                  // actually create the patches
-  for (; cell != this->dofs->end();)
+  if (data.n_datasets > 0)
     {
-      Assert (patch != this->patches.end(), ExcInternalError());
-
-                                      // use ucd_to_deal map as patch
-                                      // vertices are in the old,
-                                      // unnatural ordering
-      for (unsigned int vertex=0; vertex<GeometryInfo<DH::dimension>::vertices_per_cell; ++vertex)
-       patch->vertices[vertex] = data.mapping->transform_unit_to_real_cell
-                                 (cell, GeometryInfo<DH::dimension>::unit_cell_vertex (vertex));
+      data.x_fe_values.reinit (cell_and_index->first);
+      const FEValues<DH::dimension,DH::space_dimension> &fe_patch_values
+       = data.x_fe_values.get_present_fe_values ();
+
+      const unsigned int n_q_points = fe_patch_values.n_quadrature_points;
       
-      if (data.n_datasets > 0)
+                                      // depending on the requested output
+                                      // of curved cells, if necessary
+                                      // append the quadrature points to
+                                      // the last rows of the patch.data
+                                      // member. THis is the case if we
+                                      // want to produce curved cells at
+                                      // the boundary and this cell
+                                      // actually is at the boundary, or
+                                      // else if we want to produce curved
+                                      // cells everywhere
+      if (curved_cell_region==curved_inner_cells ||
+         (curved_cell_region==curved_boundary && cell_and_index->first->at_boundary()))
        {
-          x_fe_patch_values.reinit (cell);
-          const FEValues<DH::dimension,DH::space_dimension> &fe_patch_values
-            = x_fe_patch_values.get_present_fe_values ();
-         
-                                          // depending on the requested output
-                                          // of curved cells, if necessary
-                                          // append the quadrature points to
-                                          // the last rows of the patch->data
-                                          // member. THis is the case if we
-                                          // want to produce curved cells at
-                                          // the boundary and this cell
-                                          // actually is at the boundary, or
-                                          // else if we want to produce curved
-                                          // cells everywhere
-         if (curved_cell_region==curved_inner_cells ||
-             (curved_cell_region==curved_boundary && cell->at_boundary()))
-           {
-             Assert(patch->space_dim==dim, ExcInternalError());
-             const std::vector<Point<DH::space_dimension> > & q_points=fe_patch_values.get_quadrature_points();
-                                              // resize the patch->data member
-                                              // in order to have enough memory
-                                              // for the quadrature points as
-                                              // well
-             patch->data.reinit(patch->data.size(0)+dim,patch->data.size(1));
-                                              // set the flag indicating that
-                                              // for this cell the points are
-                                              // explicitly given
-             patch->points_are_available=true;
-                                              // copy points to patch->data
-             for (unsigned int i=0; i<dim; ++i)
-               for (unsigned int q=0; q<n_q_points; ++q)
-                 patch->data(patch->data.size(0)-dim+i,q)=q_points[q][i];
-           }
+         Assert(patch.space_dim==dim, ExcInternalError());
+         const std::vector<Point<DH::space_dimension> > & q_points=fe_patch_values.get_quadrature_points();
+                                          // resize the patch.data member
+                                          // in order to have enough memory
+                                          // for the quadrature points as
+                                          // well
+         patch.data.reinit (data.n_datasets+DH::space_dimension, n_q_points);
+                                          // set the flag indicating that
+                                          // for this cell the points are
+                                          // explicitly given
+         patch.points_are_available=true;
+                                          // copy points to patch.data
+         for (unsigned int i=0; i<DH::space_dimension; ++i)
+           for (unsigned int q=0; q<n_q_points; ++q)
+             patch.data(patch.data.size(0)-dim+i,q)=q_points[q][i];
+       }
+      else
+       {
+         patch.data.reinit(data.n_datasets, n_q_points);
+         patch.points_are_available = false;
+       }
+      
 
-                                          // counter for data records
-         unsigned int offset=0;
+                                      // counter for data records
+      unsigned int offset=0;
          
-                                          // first fill dof_data
-         for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
-           {
-             const DataPostprocessor<DH::space_dimension> *postprocessor=this->dof_data[dataset]->postprocessor;
+                                      // first fill dof_data
+      for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+       {
+         const DataPostprocessor<DH::space_dimension> *postprocessor=this->dof_data[dataset]->postprocessor;
              
-             if (postprocessor != 0)
+         if (postprocessor != 0)
+           {
+                                              // we have to postprocess the
+                                              // data, so determine, which
+                                              // fields have to be updated
+             const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
+             if (data.n_components == 1)
                {
-                                                  // we have to postprocess the
-                                                  // data, so determine, which
-                                                  // fields have to be updated
-                 const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
-                 if (data.n_components == 1)
-                   {
-                                                      // at each point there is
-                                                      // only one component of
-                                                      // value, gradient etc.
-                     if (update_flags & update_values)
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values);
-                     if (update_flags & update_gradients)
-                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                        data.patch_gradients);
-                     if (update_flags & update_hessians)
-                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                       data.patch_hessians);
-                     postprocessor->
-                       compute_derived_quantities_scalar(data.patch_values,
-                                                         data.patch_gradients,
-                                                         data.patch_hessians,
-                                                         data.dummy_normals,
-                                                         data.postprocessed_values[dataset]);
-                   }
-                 else
-                   {
-                                                      // at each point there is
-                                                      // a vector valued
-                                                      // function and its
-                                                      // derivative...
-                     if (update_flags & update_values)
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values_system);
-                     if (update_flags & update_gradients)
-                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                        data.patch_gradients_system);
-                     if (update_flags & update_hessians)
-                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                       data.patch_hessians_system);
-                     postprocessor->
-                       compute_derived_quantities_vector(data.patch_values_system,
-                                                         data.patch_gradients_system,
-                                                         data.patch_hessians_system,
-                                                         data.dummy_normals,
-                                                         data.postprocessed_values[dataset]);
-                   }
-                 
-                 for (unsigned int q=0; q<n_q_points; ++q)
-                   for (unsigned int component=0; component<this->dof_data[dataset]->n_output_variables;++component)
-                     patch->data(offset+component,q)= data.postprocessed_values[dataset][q](component);
-               }
-             else
-                                                // now we use the given data
-                                                // vector without
-                                                // modifications. again, we
-                                                // treat single component
-                                                // functions separately for
-                                                // efficiency reasons.
-               if (data.n_components == 1)
-                 {
+                                                  // at each point there is
+                                                  // only one component of
+                                                  // value, gradient etc.
+                 if (update_flags & update_values)
                    this->dof_data[dataset]->get_function_values (fe_patch_values,
                                                                  data.patch_values);
-                   for (unsigned int q=0; q<n_q_points; ++q)
-                     patch->data(offset,q) = data.patch_values[q];
-                 }
-               else
-                 {
+                 if (update_flags & update_gradients)
+                   this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                    data.patch_gradients);
+                 if (update_flags & update_hessians)
+                   this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                   data.patch_hessians);
+                 std::vector<Point<DH::space_dimension> > dummy_normals;
+                 postprocessor->
+                   compute_derived_quantities_scalar(data.patch_values,
+                                                     data.patch_gradients,
+                                                     data.patch_hessians,
+                                                     dummy_normals,
+                                                     data.postprocessed_values[dataset]);
+               }
+             else
+               {
+                                                  // at each point there is
+                                                  // a vector valued
+                                                  // function and its
+                                                  // derivative...
+                 if (update_flags & update_values)
                    this->dof_data[dataset]->get_function_values (fe_patch_values,
                                                                  data.patch_values_system);
-                   for (unsigned int component=0; component<data.n_components;
-                        ++component)
-                     for (unsigned int q=0; q<n_q_points; ++q)
-                       patch->data(offset+component,q) =
-                         data.patch_values_system[q](component);
-                 }
-                                              // increment the counter for the
-                                              // actual data record
-             offset+=this->dof_data[dataset]->n_output_variables;
+                 if (update_flags & update_gradients)
+                   this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                    data.patch_gradients_system);
+                 if (update_flags & update_hessians)
+                   this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                   data.patch_hessians_system);
+                 std::vector<Point<DH::space_dimension> > dummy_normals;
+                 postprocessor->
+                   compute_derived_quantities_vector(data.patch_values_system,
+                                                     data.patch_gradients_system,
+                                                     data.patch_hessians_system,
+                                                     dummy_normals,
+                                                     data.postprocessed_values[dataset]);
+               }
+                 
+             for (unsigned int q=0; q<n_q_points; ++q)
+               for (unsigned int component=0;
+                    component<this->dof_data[dataset]->n_output_variables;
+                    ++component)
+                 patch.data(offset+component,q)
+                   = data.postprocessed_values[dataset][q](component);
            }
+         else
+                                            // now we use the given data
+                                            // vector without
+                                            // modifications. again, we
+                                            // treat single component
+                                            // functions separately for
+                                            // efficiency reasons.
+           if (data.n_components == 1)
+             {
+               this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                             data.patch_values);
+               for (unsigned int q=0; q<n_q_points; ++q)
+                 patch.data(offset,q) = data.patch_values[q];
+             }
+           else
+             {
+               this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                             data.patch_values_system);
+               for (unsigned int component=0; component<data.n_components;
+                    ++component)
+                 for (unsigned int q=0; q<n_q_points; ++q)
+                   patch.data(offset+component,q) =
+                     data.patch_values_system[q](component);
+             }
+                                          // increment the counter for the
+                                          // actual data record
+         offset+=this->dof_data[dataset]->n_output_variables;
+       }
 
-                                          // then do the cell data. only
-                                          // compute the number of a cell if
-                                          // needed; also make sure that we
-                                          // only access cell data if the
-                                          // first_cell/next_cell functions
-                                          // only return active cells
-          if (this->cell_data.size() != 0)
-            {
-              Assert (!cell->has_children(), ExcNotImplemented());
+                                      // then do the cell data. only
+                                      // compute the number of a cell if
+                                      // needed; also make sure that we
+                                      // only access cell data if the
+                                      // first_cell/next_cell functions
+                                      // only return active cells
+      if (this->cell_data.size() != 0)
+       {
+         Assert (!cell_and_index->first->has_children(), ExcNotImplemented());
               
-              for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
-                {
-                  const double value
-                    = this->cell_data[dataset]->get_cell_data_value (cell_index);
-                  for (unsigned int q=0; q<n_q_points; ++q)
-                    patch->data(offset+dataset,q) =
-                      value;
-                }
-            }
+         for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
+           {
+             const double value
+               = this->cell_data[dataset]->get_cell_data_value (cell_and_index->second);
+             for (unsigned int q=0; q<n_q_points; ++q)
+               patch.data(offset+dataset,q) = value;
+           }
        }
+    }
 
 
-      for (unsigned int f=0; f<GeometryInfo<DH::dimension>::faces_per_cell; ++f)
-        {
-                                           // let's look up whether
-                                           // the neighbor behind that
-                                           // face is noted in the
-                                           // table of cells which we
-                                           // treat. this can only
-                                           // happen if the neighbor
-                                           // exists, and is on the
-                                           // same level as this cell,
-                                           // but it may also happen
-                                           // that the neighbor is not
-                                           // a member of the range of
-                                           // cells over which we
-                                           // loop, in which case the
-                                           // respective entry in the
-                                           // cell_to_patch_index_map
-                                           // will have the value
-                                           // no_neighbor. (note that
-                                           // since we allocated only
-                                           // as much space in this
-                                           // array as the maximum
-                                           // index of the cells we
-                                           // loop over, not every
-                                           // neighbor may have its
-                                           // space in it, so we have
-                                           // to assume that it is
-                                           // extended by values
-                                           // no_neighbor)
-          if (cell->at_boundary(f)
-              ||
-              (cell->neighbor(f)->level() != cell->level()))
-            continue;
-
-          const cell_iterator neighbor = cell->neighbor(f);
-          Assert (static_cast<unsigned int>(neighbor->level()) <
-                  data.cell_to_patch_index_map->size(),
-                  ExcInternalError());
-          if ((static_cast<unsigned int>(neighbor->index()) >=
-               (*data.cell_to_patch_index_map)[neighbor->level()].size())
-              ||
-              ((*data.cell_to_patch_index_map)[neighbor->level()][neighbor->index()]
-               ==
-               dealii::DataOutBase::Patch<DH::dimension>::no_neighbor))
-            continue;
-
-                                           // now, there is a
-                                           // neighbor, so get its
-                                           // patch number and set it
-                                           // for the neighbor index
-          patch->neighbors[f] = this->patches[(*data.cell_to_patch_index_map)
-                                             [neighbor->level()][neighbor->index()]].patch_index;
-        }
-      
-                                      // next cell (patch) in this
-                                      // thread
-      for (unsigned int i=0;
-          (i<data.n_threads)&&(cell != this->dofs->end()); ++i)
+  for (unsigned int f=0; f<GeometryInfo<DH::dimension>::faces_per_cell; ++f)
+    {
+                                      // let's look up whether
+                                      // the neighbor behind that
+                                      // face is noted in the
+                                      // table of cells which we
+                                      // treat. this can only
+                                      // happen if the neighbor
+                                      // exists, and is on the
+                                      // same level as this cell,
+                                      // but it may also happen
+                                      // that the neighbor is not
+                                      // a member of the range of
+                                      // cells over which we
+                                      // loop, in which case the
+                                      // respective entry in the
+                                      // cell_to_patch_index_map
+                                      // will have the value
+                                      // no_neighbor. (note that
+                                      // since we allocated only
+                                      // as much space in this
+                                      // array as the maximum
+                                      // index of the cells we
+                                      // loop over, not every
+                                      // neighbor may have its
+                                      // space in it, so we have
+                                      // to assume that it is
+                                      // extended by values
+                                      // no_neighbor)
+      if (cell_and_index->first->at_boundary(f)
+         ||
+         (cell_and_index->first->neighbor(f)->level() != cell_and_index->first->level()))
        {
-         ++patch;
-
-         const cell_iterator old_cell = cell;
-         
-          cell = next_cell(cell);
+         patch.neighbors[f] = numbers::invalid_unsigned_int;
+         continue;
+       }
 
-         if (this->cell_data.size() != 0)
-           cell_index += std::distance (active_cell_iterator(old_cell),
-                                        active_cell_iterator(cell));
+      const cell_iterator neighbor = cell_and_index->first->neighbor(f);
+      Assert (static_cast<unsigned int>(neighbor->level()) <
+             data.cell_to_patch_index_map->size(),
+             ExcInternalError());
+      if ((static_cast<unsigned int>(neighbor->index()) >=
+          (*data.cell_to_patch_index_map)[neighbor->level()].size())
+         ||
+         ((*data.cell_to_patch_index_map)[neighbor->level()][neighbor->index()]
+          ==
+          dealii::DataOutBase::Patch<DH::dimension>::no_neighbor))
+       {
+         patch.neighbors[f] = numbers::invalid_unsigned_int;     
+         continue;
        }
+
+                                      // now, there is a
+                                      // neighbor, so get its
+                                      // patch number and set it
+                                      // for the neighbor index
+      patch.neighbors[f]
+       = (*data.cell_to_patch_index_map)[neighbor->level()][neighbor->index()];
     }
 }
 
 
 
 template <int dim, class DH>
-void DataOut<dim,DH>::build_patches (const unsigned int n_subdivisions,
-                                    const unsigned int n_threads_) 
+void DataOut<dim,DH>::build_patches (const unsigned int n_subdivisions) 
 {
   build_patches (StaticMappingQ1<DH::dimension,DH::space_dimension>::mapping,
-                n_subdivisions, n_threads_, no_curved_cells);
+                n_subdivisions, no_curved_cells);
 }
 
 
 template <int dim, class DH>
 void DataOut<dim,DH>::build_patches (const Mapping<DH::dimension,DH::space_dimension> &mapping,
                                     const unsigned int nnnn_subdivisions,
-                                    const unsigned int n_threads_,
                                     const CurvedCellRegion curved_region) 
 {
-  unsigned int n_subdivisions = (nnnn_subdivisions != 0)
-                               ? nnnn_subdivisions
-                               : this->default_subdivisions;
-                                  // store the region in which cells shall be
-                                  // curved. If only one subdivision is
-                                  // requested then there is no need to do this
-                                  // at all
-  curved_cell_region=curved_region;
-  if (n_subdivisions<2)
-    curved_cell_region=no_curved_cells;
+                                  // Check consistency of redundant
+                                  // template parameter
+  Assert (dim==DH::dimension, ExcDimensionMismatch(dim, DH::dimension));
   
-  Assert (n_subdivisions >= 1,
-         ExcInvalidNumberOfSubdivisions(n_subdivisions));
-
   typedef DataOut_DoFData<DH, DH::dimension, DH::space_dimension> BaseClass;
   Assert (this->dofs != 0, typename BaseClass::ExcNoDoFHandlerSelected());
-
-  Assert (!DEAL_II_USE_MT || (n_threads_ >= 1),
-         ExcMessage ("Must run with at least one thread!"));
-  
-  const unsigned int n_threads = (DEAL_II_USE_MT ? n_threads_ : 1);
-
-                                  // before we start the loop:
-                                  // create a quadrature rule that
-                                  // actually has the points on this
-                                  // patch
-  QTrapez<1>     q_trapez;
-  QIterated<DH::dimension> patch_points (q_trapez, n_subdivisions);
-  
-  const unsigned int n_q_points     = patch_points.n_quadrature_points;
-  const unsigned int n_components   = this->dofs->get_fe().n_components();
-  unsigned int n_datasets=this->cell_data.size();
-  for (unsigned int i=0; i<this->dof_data.size(); ++i)
-    n_datasets+= this->dof_data[i]->n_output_variables;
-  
-                                  // clear the patches array
-  if (true)
-    {
-      std::vector< dealii::DataOutBase::Patch<DH::dimension, DH::space_dimension> > dummy;
-      this->patches.swap (dummy);
-    };
   
+  const unsigned int n_subdivisions = (nnnn_subdivisions != 0)
+                                     ? nnnn_subdivisions
+                                     : this->default_subdivisions;
+  Assert (n_subdivisions >= 1,
+         ExcInvalidNumberOfSubdivisions(n_subdivisions));
+
                                   // first count the cells we want to
                                   // create patches of. also fill the
                                   // object that maps the cell
@@ -1012,79 +957,91 @@ void DataOut<dim,DH>::build_patches (const Mapping<DH::dimension,DH::space_dimen
       
       cell_to_patch_index_map[l].resize (max_index+1,
                                          dealii::DataOutBase::Patch<DH::dimension,DH::space_dimension>::no_neighbor);
-    };
+    }
                                   
-  unsigned int n_patches = 0;
-  for (cell_iterator cell=first_cell(); cell != this->dofs->end();
-       cell = next_cell(cell))
-    {
-      Assert (static_cast<unsigned int>(cell->level()) <
-              cell_to_patch_index_map.size(),
-              ExcInternalError());
-      Assert (static_cast<unsigned int>(cell->index()) <
-              cell_to_patch_index_map[cell->level()].size(),
-              ExcInternalError());
-      
-      cell_to_patch_index_map[cell->level()][cell->index()] = n_patches;
-      ++n_patches;
-    };
-
-                                  // create the patches with default
-                                  // values. allocate as many patches
-                                  // as are needed, as this reduces
-                                  // expensive copying when push_back
-                                  // or similar operations are used
-                                  // which would regularly overflow
-                                  // the allocated amount of memory
-                                   //
-                                   // then number the patches
-                                   // consecutively
-  dealii::DataOutBase::Patch<DH::dimension,DH::space_dimension>  default_patch;
-  default_patch.n_subdivisions = n_subdivisions;
-  default_patch.data.reinit (n_datasets, n_q_points);
-  this->patches.insert (this->patches.end(), n_patches, default_patch);
-
-  for (unsigned int i=0; i<this->patches.size(); ++i)
-    this->patches[i].patch_index = i;
+  std::vector<std::pair<cell_iterator, unsigned int> > all_cells;
+  {
+    unsigned int index = 0;
+    for (cell_iterator cell=first_cell(); cell != this->dofs->end();
+        cell = next_cell(cell), ++index)
+      {
+       Assert (static_cast<unsigned int>(cell->level()) <
+               cell_to_patch_index_map.size(),
+               ExcInternalError());
+       Assert (static_cast<unsigned int>(cell->index()) <
+               cell_to_patch_index_map[cell->level()].size(),
+               ExcInternalError());
+       
+       cell_to_patch_index_map[cell->level()][cell->index()] = all_cells.size();
+       
+       all_cells.push_back (std::make_pair(cell, index));
+      }
+  }
   
-
-                                  // init data for the threads    
-  std::vector<internal::DataOut::ParallelData<DH::dimension, DH::space_dimension> > thread_data(n_threads);
-  for (unsigned int i=0;i<n_threads;++i)
-    {
-      thread_data[i].n_threads      = n_threads;
-      thread_data[i].this_thread    = i;
-      thread_data[i].n_components   = n_components;
-      thread_data[i].n_datasets     = n_datasets;
-      thread_data[i].n_subdivisions = n_subdivisions;
-      thread_data[i].patch_values.resize (n_q_points);
-      thread_data[i].patch_values_system.resize (n_q_points);
-      thread_data[i].patch_gradients.resize (n_q_points);
-      thread_data[i].patch_gradients_system.resize (n_q_points);
-      thread_data[i].patch_hessians.resize (n_q_points);
-      thread_data[i].patch_hessians_system.resize (n_q_points);
-      thread_data[i].dummy_normals.clear();
-      thread_data[i].postprocessed_values.resize(this->dof_data.size());
-      thread_data[i].mapping        = &mapping;
-
-      thread_data[i].cell_to_patch_index_map = &cell_to_patch_index_map;
-      
-      for (unsigned int k=0; k<n_q_points; ++k)
-       {
-         thread_data[i].patch_values_system[k].reinit(n_components);
-         thread_data[i].patch_gradients_system[k].resize(n_components);
-         thread_data[i].patch_hessians_system[k].resize(n_components);
-       }
-
-      for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
-       if (this->dof_data[dataset]->postprocessor)
-         thread_data[i].postprocessed_values[dataset].resize(n_q_points,Vector<double>(this->dof_data[dataset]->n_output_variables));
-    }
-
-  Threads::ThreadGroup<> threads;  
-  for (unsigned int l=0; l<n_threads; ++l)
-    threads += Threads::spawn (*this, &DataOut<dim,DH>::build_some_patches)(thread_data[l]);
-  threads.join_all();
+  this->patches.clear ();
+  this->patches.reserve (all_cells.size());
+  Assert (this->patches.size() == 0, ExcInternalError());  
+
+                                  // now create a default patch and a
+                                  // default object for the
+                                  // WorkStream object to work with
+  const QTrapez<1>     q_trapez;
+  const QIterated<DH::dimension> patch_points (q_trapez, n_subdivisions);
+  
+  const unsigned int n_components   = this->dofs->get_fe().n_components();
+  unsigned int n_datasets=this->cell_data.size();
+  for (unsigned int i=0; i<this->dof_data.size(); ++i)
+    n_datasets += this->dof_data[i]->n_output_variables;
+  
+  std::vector<unsigned int> n_postprocessor_outputs (this->dof_data.size());
+  for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+    if (this->dof_data[dataset]->postprocessor)
+      n_postprocessor_outputs[dataset] = this->dof_data[dataset]->n_output_variables;
+    else
+      n_postprocessor_outputs[dataset] = 0;
+
+  const CurvedCellRegion curved_cell_region
+    = (n_subdivisions<2 ? no_curved_cells : curved_region);
+  
+  UpdateFlags update_flags = update_values;
+  if (curved_cell_region != no_curved_cells)
+    update_flags |= update_quadrature_points;
+  
+  for (unsigned int i=0; i<this->dof_data.size(); ++i)
+    if (this->dof_data[i]->postprocessor)
+      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
+                                  // perhaps update_normal_vectors is present,
+                                  // which would only be useful on faces, but
+                                  // we may not use it here.
+  Assert (!(update_flags & update_normal_vectors),
+         ExcMessage("The update of normal vectors may not be requested for evaluation of "
+                    "data on cells via DataPostprocessor."));
+  
+  internal::DataOut::ParallelData<DH::dimension, DH::space_dimension>
+    thread_data (patch_points,
+                n_components, n_datasets, n_subdivisions,
+                n_postprocessor_outputs,
+                mapping,
+                cell_to_patch_index_map,
+                this->dofs->get_fe(),
+                update_flags);
+
+  DataOutBase::Patch<DH::dimension, DH::space_dimension> sample_patch;
+  sample_patch.n_subdivisions = n_subdivisions;
+  sample_patch.data.reinit (n_datasets, patch_points.n_quadrature_points);
+
+
+
+                                  // now build the patches in parallel
+  WorkStream::run (&all_cells[0],
+                  &all_cells[0]+all_cells.size(),
+                  std_cxx1x::bind(&DataOut<dim,DH>::build_one_patch,
+                                  *this, _1, _2, _3,
+                                  curved_cell_region),
+                  std_cxx1x::bind(&internal::DataOut::append_patch_to_list<dim,DH::space_dimension>,
+                                  _1, std_cxx1x::ref(this->patches)),
+                  thread_data,
+                  sample_patch);
 }
 
 
@@ -1177,6 +1134,7 @@ INSTANTIATE_VECTORS(hp::DoFHandler,deal_II_dimension,deal_II_dimension,deal_II_d
 
 template class DataOut<deal_II_dimension, hp::DoFHandler<deal_II_dimension> >;
 
+
 #if deal_II_dimension == 2 || deal_II_dimension ==1
 // now do actual instantiations, first for DoFHandler...
 template class DataOut_DoFData<DoFHandler<deal_II_dimension,deal_II_dimension+1>,deal_II_dimension, deal_II_dimension+1>;
index b4e78651098ee2d3e90afb86944ecf13aaabc7f3..399a7af9b8917468180ecfb7349b98e99e672074 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -13,7 +13,7 @@
 
 
 #include <base/quadrature_lib.h>
-#include <base/thread_management.h>
+#include <base/work_stream.h>
 #include <lac/vector.h>
 #include <lac/block_vector.h>
 #include <numerics/data_out_faces.h>
 DEAL_II_NAMESPACE_OPEN
 
 
-
-template <int dim, class DH>
-void DataOutFaces<dim,DH>::build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &data)
+namespace internal
 {
-                                  // Check consistency of redundant
-                                  // template parameter
-  Assert (dim==DH::dimension, ExcDimensionMismatch(dim, DH::dimension));
-  
-  QTrapez<1>        q_trapez;
-  QIterated<DH::dimension-1>  patch_points (q_trapez, data.n_subdivisions);
-  
-//TODO[?]: This is strange -- Data has a member 'mapping' that should
-//be used here, but it isn't. Rather, up until version 1.94, we were
-//actually initializing a local mapping object and used that... While
-//we use the mapping to transform the vertex coordinates, we do not
-//use the mapping to transform the shape functions (necessary for
-//example for Raviart-Thomas elements). This could lead to trouble
-//when someone tries to use MappingEulerian with such elements
-
-                                  // create collection objects from
-                                  // single quadratures,
-                                  // and finite elements. if we have
-                                  // an hp DoFHandler,
-                                  // dof_handler.get_fe() returns a
-                                  // collection of which we do a
-                                  // shallow copy instead
-  const hp::QCollection<DH::dimension-1>     q_collection (patch_points);
-  const hp::FECollection<DH::dimension>      fe_collection(this->dofs->get_fe());
-  
-  UpdateFlags update_flags=update_values;
-  for (unsigned int i=0; i<this->dof_data.size(); ++i)
-    if (this->dof_data[i]->postprocessor)
-      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
-  
-  hp::FEFaceValues<DH::dimension> x_fe_patch_values (fe_collection, q_collection,
-                                                    update_flags);
+  namespace DataOutFaces
+  {
+    template <int dim, int spacedim>
+    template <class FE>
+    ParallelData<dim,spacedim>::
+    ParallelData (const Quadrature<dim-1> &quadrature,
+                 const unsigned int n_components,
+                 const unsigned int n_datasets,
+                 const unsigned int n_subdivisions,
+                 const std::vector<unsigned int> &n_postprocessor_outputs,
+                 const FE &finite_elements,
+                 const UpdateFlags update_flags)
+                 :
+                 internal::DataOut::
+                 ParallelDataBase<dim,spacedim> (n_components,
+                                                 n_datasets,
+                                                 n_subdivisions,
+                                                 quadrature.n_quadrature_points,
+                                                 n_postprocessor_outputs,
+                                                 finite_elements),
+                 q_collection (quadrature),
+                 x_fe_values (this->fe_collection,
+                              q_collection,
+                              update_flags)
+    {}
+
+
+                                    /**
+                                     * In a WorkStream context, use
+                                     * this function to append the
+                                     * patch computed by the parallel
+                                     * stage to the array of patches.
+                                     */
+    template <int dim, int spacedim>
+    void
+    append_patch_to_list (const DataOutBase::Patch<dim-1,spacedim> &patch,
+                         std::vector<DataOutBase::Patch<dim-1,spacedim> > &patches)
+    {
+      patches.push_back (patch);
+      patches.back().patch_index = patches.size()-1;
+    }    
+  }
+}
 
-  const unsigned int n_q_points = patch_points.n_quadrature_points;
-  
-  typename std::vector< dealii::DataOutBase::Patch<DH::dimension-1,DH::dimension> >::iterator patch = this->patches.begin();
-  FaceDescriptor face=first_face();
 
-                                  // get first face in this thread
-  for (unsigned int i=0; (i<data.this_thread)&&(face.first != this->dofs->end()); ++i)
-    {
-      ++patch;
-      face=next_face(face);
-    }
 
-                                  // now loop over all cells and
-                                  // actually create the patches
-  for (; face.first != this->dofs->end();)
-    {
-      Assert (patch != this->patches.end(), ExcInternalError());
-      
-      for (unsigned int vertex=0; vertex<GeometryInfo<DH::dimension-1>::vertices_per_cell; ++vertex)
-       patch->vertices[vertex] = face.first->face(face.second)->vertex(vertex);
+template <int dim, class DH>
+void
+DataOutFaces<dim,DH>::
+build_one_patch (const FaceDescriptor *cell_and_face,
+                internal::DataOutFaces::ParallelData<DH::dimension, DH::dimension> &data,
+                DataOutBase::Patch<DH::dimension-1,DH::space_dimension>  &patch)
+{
+  for (unsigned int vertex=0; vertex<GeometryInfo<DH::dimension-1>::vertices_per_cell; ++vertex)
+    patch.vertices[vertex] = cell_and_face->first->face(cell_and_face->second)->vertex(vertex);
       
-      if (data.n_datasets > 0)
-       {
-         x_fe_patch_values.reinit (face.first, face.second);
-          const FEFaceValues<DH::dimension> &fe_patch_values
-            = x_fe_patch_values.get_present_fe_values ();
+  if (data.n_datasets > 0)
+    {
+      data.x_fe_values.reinit (cell_and_face->first, cell_and_face->second);
+      const FEFaceValues<DH::dimension> &fe_patch_values
+       = data.x_fe_values.get_present_fe_values ();
+
+      const unsigned int n_q_points = fe_patch_values.n_quadrature_points;
+  
          
-                                          // counter for data records
-         unsigned int offset=0;
+                                      // counter for data records
+      unsigned int offset=0;
          
-                                          // first fill dof_data
-         for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+                                      // first fill dof_data
+      for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+       {
+         const DataPostprocessor<dim> *postprocessor=this->dof_data[dataset]->postprocessor;
+         if (postprocessor != 0)
            {
-             const DataPostprocessor<dim> *postprocessor=this->dof_data[dataset]->postprocessor;
-             if (postprocessor != 0)
-               {
-                                                  // we have to postprocess the
-                                                  // data, so determine, which
-                                                  // fields have to be updated
-                 const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
-                 
-                                                  // get normals, if
-                                                  // needed. this is a
-                                                  // geometrical information
-                                                  // and thus does not depend
-                                                  // on the number of
-                                                  // components of the data
-                                                  // vector
-                 if (update_flags & update_normal_vectors)
-                   data.patch_normals=fe_patch_values.get_normal_vectors();
-
-                 if (data.n_components == 1)
-                   {
-                                                      // at each point there is
-                                                      // only one component of
-                                                      // value, gradient etc.
-                     if (update_flags & update_values)
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values);
-                     if (update_flags & update_gradients)
-                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                        data.patch_gradients);
-                     if (update_flags & update_hessians)
-                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                       data.patch_hessians);
-                     postprocessor->
-                       compute_derived_quantities_scalar(data.patch_values,
-                                                         data.patch_gradients,
-                                                         data.patch_hessians,
-                                                         data.patch_normals,
-                                                         data.postprocessed_values[dataset]);
-                   }
-                 else
-                   {
-                                                      // at each point there is
-                                                      // a vector valued
-                                                      // function and its
-                                                      // derivative...
-                     if (update_flags & update_values)
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values_system);
-                     if (update_flags & update_gradients)
-                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                        data.patch_gradients_system);
-                     if (update_flags & update_hessians)
-                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                       data.patch_hessians_system);
-                     postprocessor->
-                       compute_derived_quantities_vector(data.patch_values_system,
-                                                         data.patch_gradients_system,
-                                                         data.patch_hessians_system,
-                                                         data.patch_normals,
-                                                         data.postprocessed_values[dataset]);
-                   }
+                                              // we have to postprocess the
+                                              // data, so determine, which
+                                              // fields have to be updated
+             const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
                  
-                 for (unsigned int q=0; q<n_q_points; ++q)
-                   for (unsigned int component=0; component<this->dof_data[dataset]->n_output_variables;++component)
-                     patch->data(offset+component,q)= data.postprocessed_values[dataset][q](component);
-               }
-             else
-                                                // now we use the given data
-                                                // vector without
-                                                // modifications. again, we
-                                                // treat single component
-                                                // functions separately for
-                                                // efficiency reasons.
-               if (data.n_components == 1)
-                 {
+                                              // get normals, if
+                                              // needed. this is a
+                                              // geometrical information
+                                              // and thus does not depend
+                                              // on the number of
+                                              // components of the data
+                                              // vector
+             if (update_flags & update_normal_vectors)
+               data.patch_normals=fe_patch_values.get_normal_vectors();
+
+             if (data.n_components == 1)
+               {
+                                                  // at each point there is
+                                                  // only one component of
+                                                  // value, gradient etc.
+                 if (update_flags & update_values)
                    this->dof_data[dataset]->get_function_values (fe_patch_values,
                                                                  data.patch_values);
-                   for (unsigned int q=0; q<n_q_points; ++q)
-                     patch->data(offset,q) = data.patch_values[q];
-                 }
-               else
-                 {
+                 if (update_flags & update_gradients)
+                   this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                    data.patch_gradients);
+                 if (update_flags & update_hessians)
+                   this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                   data.patch_hessians);
+                 postprocessor->
+                   compute_derived_quantities_scalar(data.patch_values,
+                                                     data.patch_gradients,
+                                                     data.patch_hessians,
+                                                     data.patch_normals,
+                                                     data.postprocessed_values[dataset]);
+               }
+             else
+               {
+                                                  // at each point there is
+                                                  // a vector valued
+                                                  // function and its
+                                                  // derivative...
+                 if (update_flags & update_values)
                    this->dof_data[dataset]->get_function_values (fe_patch_values,
                                                                  data.patch_values_system);
-                   for (unsigned int component=0; component<data.n_components;
-                        ++component)
-                     for (unsigned int q=0; q<n_q_points; ++q)
-                       patch->data(offset+component,q) =
-                         data.patch_values_system[q](component);
-                 }
-                                              // increment the counter for the
-                                              // actual data record
-             offset+=this->dof_data[dataset]->n_output_variables;
-           }
-
-                                          // then do the cell data
-         for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
-           {
-                                              // we need to get at
-                                              // the number of the
-                                              // cell to which this
-                                              // face belongs in
-                                              // order to access the
-                                              // cell data. this is
-                                              // not readily
-                                              // available, so choose
-                                              // the following rather
-                                              // inefficient way:
-             Assert (face.first->active(), ExcCellNotActiveForCellData());
-             const unsigned int cell_number
-               = std::distance (this->dofs->begin_active(),
-                                typename DH::active_cell_iterator(face.first));
-
-              const double value
-                = this->cell_data[dataset]->get_cell_data_value (cell_number);
-              for (unsigned int q=0; q<n_q_points; ++q)
-                patch->data(dataset+offset,q) =
-                  value;
+                 if (update_flags & update_gradients)
+                   this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                    data.patch_gradients_system);
+                 if (update_flags & update_hessians)
+                   this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                   data.patch_hessians_system);
+                 postprocessor->
+                   compute_derived_quantities_vector(data.patch_values_system,
+                                                     data.patch_gradients_system,
+                                                     data.patch_hessians_system,
+                                                     data.patch_normals,
+                                                     data.postprocessed_values[dataset]);
+               }
+                 
+             for (unsigned int q=0; q<n_q_points; ++q)
+               for (unsigned int component=0;
+                    component<this->dof_data[dataset]->n_output_variables;++component)
+                 patch.data(offset+component,q)
+                   = data.postprocessed_values[dataset][q](component);
            }
+         else
+                                            // now we use the given data
+                                            // vector without
+                                            // modifications. again, we
+                                            // treat single component
+                                            // functions separately for
+                                            // efficiency reasons.
+           if (data.n_components == 1)
+             {
+               this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                             data.patch_values);
+               for (unsigned int q=0; q<n_q_points; ++q)
+                 patch.data(offset,q) = data.patch_values[q];
+             }
+           else
+             {
+               this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                             data.patch_values_system);
+               for (unsigned int component=0; component<data.n_components;
+                    ++component)
+                 for (unsigned int q=0; q<n_q_points; ++q)
+                   patch.data(offset+component,q) =
+                     data.patch_values_system[q](component);
+             }
+                                          // increment the counter for the
+                                          // actual data record
+         offset+=this->dof_data[dataset]->n_output_variables;
        }
-                                      // next cell (patch) in this thread
-      for (unsigned int i=0;
-          (i<data.n_threads)&&(face.first != this->dofs->end()); ++i)
+
+                                      // then do the cell data
+      for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
        {
-         ++patch;
-         face=next_face(face);
+                                          // we need to get at
+                                          // the number of the
+                                          // cell to which this
+                                          // face belongs in
+                                          // order to access the
+                                          // cell data. this is
+                                          // not readily
+                                          // available, so choose
+                                          // the following rather
+                                          // inefficient way:
+         Assert (cell_and_face->first->active(), ExcCellNotActiveForCellData());
+         const unsigned int cell_number
+           = std::distance (this->dofs->begin_active(),
+                            typename DH::active_cell_iterator(cell_and_face->first));
+
+         const double value
+           = this->cell_data[dataset]->get_cell_data_value (cell_number);
+         for (unsigned int q=0; q<n_q_points; ++q)
+           patch.data(dataset+offset,q) = value;
        }
     }
 }
@@ -233,12 +229,15 @@ void DataOutFaces<dim,DH>::build_some_patches (internal::DataOut::ParallelData<D
 
 
 template <int dim, class DH>
-void DataOutFaces<dim,DH>::build_patches (const unsigned int nnnn_subdivisions,
-                                         const unsigned int n_threads_) 
+void DataOutFaces<dim,DH>::build_patches (const unsigned int n_subdivisions_)
 {
-  unsigned int n_subdivisions = (nnnn_subdivisions != 0)
-                               ? nnnn_subdivisions
-                               : this->default_subdivisions;
+                                  // Check consistency of redundant
+                                  // template parameter
+  Assert (dim==DH::dimension, ExcDimensionMismatch(dim, DH::dimension));
+
+  const unsigned int n_subdivisions = (n_subdivisions_ != 0)
+                                     ? n_subdivisions_
+                                     : this->default_subdivisions;
   
   Assert (n_subdivisions >= 1,
          ExcInvalidNumberOfSubdivisions(n_subdivisions));
@@ -246,89 +245,68 @@ void DataOutFaces<dim,DH>::build_patches (const unsigned int nnnn_subdivisions,
   typedef DataOut_DoFData<DH,DH::dimension+1> BaseClass;
   Assert (this->dofs != 0, typename BaseClass::ExcNoDoFHandlerSelected());
 
-  Assert (!DEAL_II_USE_MT || (n_threads_ >= 1),
-         ExcMessage ("Must run with at least one thread!"));
-  const unsigned int n_threads = (DEAL_II_USE_MT ? n_threads_ : 1);
-
                                   // before we start the loop:
                                   // create a quadrature rule that
                                   // actually has the points on this
                                   // patch
-  QTrapez<1>       q_trapez;
-  QIterated<DH::dimension-1> patch_points (q_trapez, n_subdivisions);
+  const QTrapez<1>       q_trapez;
+  const QIterated<DH::dimension-1> patch_points (q_trapez, n_subdivisions);
 
-  const unsigned int n_q_points     = patch_points.n_quadrature_points;
   const unsigned int n_components   = this->dofs->get_fe().n_components();
+
   unsigned int n_datasets     = this->cell_data.size();
   for (unsigned int i=0; i<this->dof_data.size(); ++i)
-    n_datasets+= this->dof_data[i]->n_output_variables;
-  
-                                  // clear the patches array
-  if (true)
-    {
-      std::vector< dealii::DataOutBase::Patch<DH::dimension-1,DH::dimension> > dummy;
-      this->patches.swap (dummy);
-    };
+    n_datasets += this->dof_data[i]->n_output_variables;
   
                                   // first count the cells we want to
                                   // create patches of and make sure
                                   // there is enough memory for that
-  unsigned int n_patches = 0;
+  std::vector<FaceDescriptor> all_faces;
   for (FaceDescriptor face=first_face();
        face.first != this->dofs->end();
        face = next_face(face))
-    ++n_patches;
-
-  std::vector<internal::DataOut::ParallelData<DH::dimension, DH::dimension> > thread_data(n_threads);
-
-                                  // init data for the threads
-  for (unsigned int i=0;i<n_threads;++i)
-    {
-      thread_data[i].n_threads            = n_threads;
-      thread_data[i].this_thread          = i;
-      thread_data[i].n_components         = n_components;
-      thread_data[i].n_datasets           = n_datasets;
-      thread_data[i].n_subdivisions       = n_subdivisions;
-      thread_data[i].patch_values.resize (n_q_points);
-      thread_data[i].patch_values_system.resize (n_q_points);
-      thread_data[i].patch_gradients.resize (n_q_points);
-      thread_data[i].patch_gradients_system.resize (n_q_points);
-      thread_data[i].patch_hessians.resize (n_q_points);
-      thread_data[i].patch_hessians_system.resize (n_q_points);
-      thread_data[i].patch_normals.resize (n_q_points);
-      thread_data[i].postprocessed_values.resize (this->dof_data.size());
-      
-      for (unsigned int k=0; k<n_q_points; ++k)
-       {
-         thread_data[i].patch_values_system[k].reinit(n_components);
-         thread_data[i].patch_gradients_system[k].resize(n_components);
-         thread_data[i].patch_hessians_system[k].resize(n_components);
-       }
-
-      for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
-       if (this->dof_data[dataset]->postprocessor)
-         thread_data[i].postprocessed_values[dataset].resize(n_q_points,Vector<double>(this->dof_data[dataset]->n_output_variables));
-    }
-
-                                  // create the patches with default
-                                  // values. note that the evaluation
-                                  // points on the face have to be
-                                  // repeated in angular direction
-  dealii::DataOutBase::Patch<DH::dimension-1,DH::dimension>  default_patch;
-  default_patch.n_subdivisions = n_subdivisions;
-  default_patch.data.reinit (n_datasets, n_q_points);
-  this->patches.insert (this->patches.end(), n_patches, default_patch);
+    all_faces.push_back (face);
+
+                                  // clear the patches array and
+                                  // allocate the right number of
+                                  // elements
+  this->patches.clear ();
+  this->patches.reserve (all_faces.size());
+  Assert (this->patches.size() == 0, ExcInternalError());  
+  
+  
+  std::vector<unsigned int> n_postprocessor_outputs (this->dof_data.size());
+  for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+    if (this->dof_data[dataset]->postprocessor)
+      n_postprocessor_outputs[dataset] = this->dof_data[dataset]->n_output_variables;
+    else
+      n_postprocessor_outputs[dataset] = 0;
 
-  if (DEAL_II_USE_MT)
-    {
-      Threads::ThreadGroup<> threads;  
-      for (unsigned int l=0;l<n_threads;++l)
-        threads += Threads::spawn (*this, &DataOutFaces<dim,DH>::build_some_patches)(thread_data[l]);
-      threads.join_all();
-    }
-  else
-                                    // just one thread
-    build_some_patches(thread_data[0]);
+  UpdateFlags update_flags=update_values;
+  for (unsigned int i=0; i<this->dof_data.size(); ++i)
+    if (this->dof_data[i]->postprocessor)
+      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
+  
+  internal::DataOutFaces::ParallelData<DH::dimension, DH::dimension>
+    thread_data (patch_points, n_components, n_datasets,
+                n_subdivisions, 
+                n_postprocessor_outputs, this->dofs->get_fe(),
+                update_flags);
+  DataOutBase::Patch<DH::dimension-1,DH::space_dimension> sample_patch;
+  sample_patch.n_subdivisions = n_subdivisions;
+  sample_patch.data.reinit (n_datasets,
+                           patch_points.n_quadrature_points);
+
+                                  // now build the patches in parallel
+  WorkStream::run (&all_faces[0],
+                  &all_faces[0]+all_faces.size(),
+                  std_cxx1x::bind(&DataOutFaces<dim,DH>::build_one_patch,
+                                  *this, _1, _2, _3),
+                  std_cxx1x::bind(&internal::DataOutFaces::
+                                  append_patch_to_list<dim,DH::space_dimension>,
+                                  _1, std_cxx1x::ref(this->patches)),
+                  thread_data,
+                  sample_patch);
 }
 
 
index 44d2202ec10382023f6f0f3bd03c993780f3ad0d..d456b165d01665e479b3f86ce1320c39ea647c1c 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -13,7 +13,7 @@
 
 
 #include <base/quadrature_lib.h>
-#include <base/thread_management.h>
+#include <base/work_stream.h>
 #include <lac/vector.h>
 #include <lac/block_vector.h>
 #include <grid/tria.h>
@@ -41,42 +41,68 @@ DEAL_II_NAMESPACE_OPEN
 
 // Not implemented for 3D
 
+
+namespace internal
+{
+  namespace DataOutRotation
+  {
+    template <int dim, int spacedim>
+    template <class FE>
+    ParallelData<dim,spacedim>::
+    ParallelData (const Quadrature<dim> &quadrature,
+                 const unsigned int n_components,
+                 const unsigned int n_datasets,
+                 const unsigned int n_subdivisions,
+                 const unsigned int n_patches_per_circle,
+                 const std::vector<unsigned int> &n_postprocessor_outputs,
+                 const FE &finite_elements,
+                 const UpdateFlags update_flags)
+                   :
+                   internal::DataOut::
+                   ParallelDataBase<dim,spacedim> (n_components,
+                                                   n_datasets,
+                                                   n_subdivisions,
+                                                   quadrature.n_quadrature_points,
+                                                   n_postprocessor_outputs,
+                                                   finite_elements),
+                   n_patches_per_circle (n_patches_per_circle),
+                   q_collection (quadrature),
+                   x_fe_values (this->fe_collection,
+                                q_collection,
+                                update_flags)
+    {}
+
+
+                                    /**
+                                     * In a WorkStream context, use
+                                     * this function to append the
+                                     * patch computed by the parallel
+                                     * stage to the array of patches.
+                                     */
+    template <int dim, int spacedim>
+    void
+    append_patch_to_list (const std::vector<DataOutBase::Patch<dim+1,spacedim+1> > &new_patches,
+                         std::vector<DataOutBase::Patch<dim+1,spacedim+1> > &patches)
+    {
+      for (unsigned int i=0; i<new_patches.size(); ++i)
+       {
+         patches.push_back (new_patches[i]);
+         patches.back().patch_index = patches.size()-1;
+       }
+    }    
+  }
+}
+
+    
+
 #if deal_II_dimension < 3
 template <int dim, class DH>
-void DataOutRotation<dim,DH>::build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &data)
+void
+DataOutRotation<dim,DH>::
+build_one_patch (const cell_iterator *cell,
+                internal::DataOutRotation::ParallelData<DH::dimension, DH::space_dimension> &data,
+                std::vector<DataOutBase::Patch<DH::dimension+1,DH::space_dimension+1> > &patches)
 {
-  QTrapez<1>     q_trapez;
-  QIterated<DH::dimension> patch_points (q_trapez, data.n_subdivisions);
-
-                                  // create collection objects from
-                                  // single quadratures,
-                                  // and finite elements. if we have
-                                  // an hp DoFHandler,
-                                  // dof_handler.get_fe() returns a
-                                  // collection of which we do a
-                                  // shallow copy instead
-                                   //
-                                  // since most output formats can't
-                                  // handle cells that are not
-                                  // transformed using a Q1 mapping,
-                                  // we don't support anything else
-                                  // as well
-  const hp::QCollection<DH::dimension>       q_collection (patch_points);
-  const hp::FECollection<DH::dimension>      fe_collection(this->dofs->get_fe());
-  
-  UpdateFlags update_flags=update_values | update_quadrature_points;
-  for (unsigned int i=0; i<this->dof_data.size(); ++i)
-    if (this->dof_data[i]->postprocessor)
-      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
-                                  // perhaps update_normal_vectors is present,
-                                  // which would only be useful on faces, but
-                                  // we may not use it here.
-  Assert (!(update_flags & update_normal_vectors),
-         ExcMessage("The update of normal vectors may not be requested for evaluation of data on cells via DataPostprocessor."));
-  
-  hp::FEValues<DH::dimension> x_fe_patch_values (fe_collection, q_collection,
-                                                update_flags);
-
   const unsigned int n_patches_per_circle = data.n_patches_per_circle;
 
                                   // another abbreviation denoting
@@ -100,303 +126,278 @@ void DataOutRotation<dim,DH>::build_some_patches (internal::DataOut::ParallelDat
                                                      i/n_patches_per_circle);
       angle_directions[i][DH::dimension] = std::sin(2*numbers::PI *
                                                    i/n_patches_per_circle);
-    };
-  
-  
-  unsigned int cell_number = 0;
-  typename std::vector< dealii::DataOutBase::Patch<DH::dimension+1> >::iterator
-    patch = this->patches.begin();
-  cell_iterator cell=first_cell();
-
-                                  // get first cell in this thread
-  for (unsigned int i=0; (i<data.this_thread)&&(cell != this->dofs->end()); ++i)
-    {
-      std::advance (patch, n_patches_per_circle);
-      ++cell_number;
-      cell=next_cell(cell);
     }
-
-                                  // now loop over all cells and
-                                  // actually create the patches
-  for (; cell != this->dofs->end(); )
+  
+  for (unsigned int angle=0; angle<n_patches_per_circle; ++angle)
     {
-      for (unsigned int angle=0; angle<n_patches_per_circle; ++angle, ++patch)
+                                      // first compute the
+                                      // vertices of the
+                                      // patch. note that they
+                                      // will have to be computed
+                                      // from the vertices of the
+                                      // cell, which has one
+                                      // dimension less, however.
+      switch (DH::dimension)
        {
-         Assert (patch != this->patches.end(), ExcInternalError());
-         
-
-                                          // first compute the
-                                          // vertices of the
-                                          // patch. note that they
-                                          // will have to be computed
-                                          // from the vertices of the
-                                          // cell, which has one
-                                          // dimension less, however.
-         switch (DH::dimension)
-           {
-             case 1:
-             {
-               const double r1 = cell->vertex(0)(0),
-                            r2 = cell->vertex(1)(0);
-               Assert (r1 >= 0, ExcRadialVariableHasNegativeValues(r1));       
-               Assert (r2 >= 0, ExcRadialVariableHasNegativeValues(r2));
+         case 1:
+         {
+           const double r1 = (*cell)->vertex(0)(0),
+                        r2 = (*cell)->vertex(1)(0);
+           Assert (r1 >= 0, ExcRadialVariableHasNegativeValues(r1));   
+           Assert (r2 >= 0, ExcRadialVariableHasNegativeValues(r2));
                
-               patch->vertices[0] = r1*angle_directions[angle];
-               patch->vertices[1] = r2*angle_directions[angle];
-               patch->vertices[2] = r1*angle_directions[angle+1];
-               patch->vertices[3] = r2*angle_directions[angle+1];
+           patches[angle].vertices[0] = r1*angle_directions[angle];
+           patches[angle].vertices[1] = r2*angle_directions[angle];
+           patches[angle].vertices[2] = r1*angle_directions[angle+1];
+           patches[angle].vertices[3] = r2*angle_directions[angle+1];
 
-               break;
-             };
+           break;
+         };
               
-             case 2:
+         case 2:
+         {
+           for (unsigned int vertex=0;
+                vertex<GeometryInfo<DH::dimension>::vertices_per_cell;
+                ++vertex)
              {
-               for (unsigned int vertex=0;
-                    vertex<GeometryInfo<DH::dimension>::vertices_per_cell;
-                    ++vertex)
-                 {
-                   const Point<DH::dimension> v = cell->vertex(vertex);
+               const Point<DH::dimension> v = (*cell)->vertex(vertex);
                    
-                                                    // make sure that the
-                                                    // radial variable does
-                                                    // attain negative
-                                                    // values
-                   Assert (v(0) >= 0, ExcRadialVariableHasNegativeValues(v(0)));
+                                                // make sure that the
+                                                // radial variable does
+                                                // attain negative
+                                                // values
+               Assert (v(0) >= 0, ExcRadialVariableHasNegativeValues(v(0)));
                    
-                                                    // now set the vertices
-                                                    // of the patch
-                   patch->vertices[vertex] = v(0) * angle_directions[angle];
-                   patch->vertices[vertex][0] = v(1);
+                                                // now set the vertices
+                                                // of the patch
+               patches[angle].vertices[vertex] = v(0) * angle_directions[angle];
+               patches[angle].vertices[vertex][0] = v(1);
                    
-                   patch->vertices[vertex+GeometryInfo<DH::dimension>::vertices_per_cell]
-                     = v(0) * angle_directions[angle+1];
-                   patch->vertices[vertex+GeometryInfo<DH::dimension>::vertices_per_cell][0]
-                     = v(1);
-                 };
-               
-               break;
+               patches[angle].vertices[vertex+GeometryInfo<DH::dimension>::vertices_per_cell]
+                 = v(0) * angle_directions[angle+1];
+               patches[angle].vertices[vertex+GeometryInfo<DH::dimension>::vertices_per_cell][0]
+                 = v(1);
              };
+               
+           break;
+         };
 
-             default:
-                   Assert (false, ExcNotImplemented());
-           };
+         default:
+               Assert (false, ExcNotImplemented());
+       };
          
-         unsigned int offset=0;
+      unsigned int offset=0;
               
-                                          // then fill in data
-         if (data.n_datasets > 0)
-           {
-             x_fe_patch_values.reinit (cell);
-              const FEValues<DH::dimension> &fe_patch_values
-                = x_fe_patch_values.get_present_fe_values ();
+                                      // then fill in data
+      if (data.n_datasets > 0)
+       {
+         data.x_fe_values.reinit (*cell);
+         const FEValues<DH::dimension> &fe_patch_values
+           = data.x_fe_values.get_present_fe_values ();
 
-                                              // first fill dof_data
-             for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+                                          // first fill dof_data
+         for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+           {
+             const DataPostprocessor<dim> *postprocessor=this->dof_data[dataset]->postprocessor;
+             if (postprocessor != 0)
                {
-                 const DataPostprocessor<dim> *postprocessor=this->dof_data[dataset]->postprocessor;
-                 if (postprocessor != 0)
-                   {
-                                                      // we have to postprocess the
-                                                      // data, so determine, which
-                                                      // fields have to be updated
-                     const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
+                                                  // we have to postprocess the
+                                                  // data, so determine, which
+                                                  // fields have to be updated
+                 const UpdateFlags update_flags=postprocessor->get_needed_update_flags();
 
-                     if (data.n_components == 1)
-                       {
-                                                          // at each point there is
-                                                          // only one component of
-                                                          // value, gradient etc.
-                         if (update_flags & update_values)
-                           this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                         data.patch_values);
-                         if (update_flags & update_gradients)
-                           this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                            data.patch_gradients);
-                         if (update_flags & update_hessians)
-                           this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                           data.patch_hessians);
-                         postprocessor->
-                           compute_derived_quantities_scalar(data.patch_values,
-                                                             data.patch_gradients,
-                                                             data.patch_hessians,
-                                                             data.dummy_normals,
-                                                             data.postprocessed_values[dataset]);
-                       }
-                     else
-                       {
-                                                          // at each point there is
-                                                          // a vector valued
-                                                          // function and its
-                                                          // derivative...
-                         if (update_flags & update_values)
-                           this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                         data.patch_values_system);
-                         if (update_flags & update_gradients)
-                           this->dof_data[dataset]->get_function_gradients (fe_patch_values,
-                                                                            data.patch_gradients_system);
-                         if (update_flags & update_hessians)
-                           this->dof_data[dataset]->get_function_hessians (fe_patch_values,
-                                                                           data.patch_hessians_system);
-                         postprocessor->
-                           compute_derived_quantities_vector(data.patch_values_system,
-                                                             data.patch_gradients_system,
-                                                             data.patch_hessians_system,
-                                                             data.dummy_normals,
-                                                             data.postprocessed_values[dataset]);
-                       }
+                 if (data.n_components == 1)
+                   {
+                                                      // at each point there is
+                                                      // only one component of
+                                                      // value, gradient etc.
+                     if (update_flags & update_values)
+                       this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                                     data.patch_values);
+                     if (update_flags & update_gradients)
+                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                        data.patch_gradients);
+                     if (update_flags & update_hessians)
+                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                       data.patch_hessians);
+
+                     std::vector<Point<DH::space_dimension> > dummy_normals;
+                     postprocessor->
+                       compute_derived_quantities_scalar(data.patch_values,
+                                                         data.patch_gradients,
+                                                         data.patch_hessians,
+                                                         dummy_normals,
+                                                         data.postprocessed_values[dataset]);
+                   }
+                 else
+                   {
+                                                      // at each point there is
+                                                      // a vector valued
+                                                      // function and its
+                                                      // derivative...
+                     if (update_flags & update_values)
+                       this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                                     data.patch_values_system);
+                     if (update_flags & update_gradients)
+                       this->dof_data[dataset]->get_function_gradients (fe_patch_values,
+                                                                        data.patch_gradients_system);
+                     if (update_flags & update_hessians)
+                       this->dof_data[dataset]->get_function_hessians (fe_patch_values,
+                                                                       data.patch_hessians_system);
+                     std::vector<Point<DH::space_dimension> > dummy_normals;
+                     postprocessor->
+                       compute_derived_quantities_vector(data.patch_values_system,
+                                                         data.patch_gradients_system,
+                                                         data.patch_hessians_system,
+                                                         dummy_normals,
+                                                         data.postprocessed_values[dataset]);
+                   }
                      
-                     for (unsigned int component=0;
-                          component<this->dof_data[dataset]->n_output_variables;
-                          ++component)
+                 for (unsigned int component=0;
+                      component<this->dof_data[dataset]->n_output_variables;
+                      ++component)
+                   {
+                     switch (DH::dimension)
                        {
-                         switch (DH::dimension)
-                           {
-                             case 1:
-                                   for (unsigned int x=0; x<n_points; ++x)
-                                     for (unsigned int y=0; y<n_points; ++y)
-                                       patch->data(offset+component,
-                                                   x*n_points + y)
-                                         = data.postprocessed_values[dataset][x](component);
-                                   break;
+                         case 1:
+                               for (unsigned int x=0; x<n_points; ++x)
+                                 for (unsigned int y=0; y<n_points; ++y)
+                                   patches[angle].data(offset+component,
+                                               x*n_points + y)
+                                     = data.postprocessed_values[dataset][x](component);
+                               break;
                                      
-                             case 2:
-                                   for (unsigned int x=0; x<n_points; ++x)
-                                     for (unsigned int y=0; y<n_points; ++y)
-                                       for (unsigned int z=0; z<n_points; ++z)
-                                         patch->data(offset+component,
-                                                     x*n_points*n_points +
-                                                     y*n_points +
-                                                     z)
-                                           = data.postprocessed_values[dataset][x*n_points+z](component);
-                                   break;
+                         case 2:
+                               for (unsigned int x=0; x<n_points; ++x)
+                                 for (unsigned int y=0; y<n_points; ++y)
+                                   for (unsigned int z=0; z<n_points; ++z)
+                                     patches[angle].data(offset+component,
+                                                 x*n_points*n_points +
+                                                 y*n_points +
+                                                 z)
+                                       = data.postprocessed_values[dataset][x*n_points+z](component);
+                               break;
                                      
-                             default:
-                                   Assert (false, ExcNotImplemented());
-                           }
+                         default:
+                               Assert (false, ExcNotImplemented());
                        }
                    }
-                 else
-                   if (data.n_components == 1)
+               }
+             else
+               if (data.n_components == 1)
+                 {
+                   this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                                 data.patch_values);
+                       
+                   switch (DH::dimension)
                      {
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values);
+                       case 1:
+                             for (unsigned int x=0; x<n_points; ++x)
+                               for (unsigned int y=0; y<n_points; ++y)
+                                 patches[angle].data(offset,
+                                             x*n_points + y)
+                                   = data.patch_values[x];
+                             break;
+                                 
+                       case 2:
+                             for (unsigned int x=0; x<n_points; ++x)
+                               for (unsigned int y=0; y<n_points; ++y)
+                                 for (unsigned int z=0; z<n_points; ++z)
+                                   patches[angle].data(offset,
+                                               x*n_points*n_points +
+                                               y +
+                                               z*n_points)
+                                     = data.patch_values[x*n_points+z];
+                             break;
+                                 
+                       default:
+                             Assert (false, ExcNotImplemented());
+                     }
+                 }
+               else
+                                                  // system of components
+                 {
+                   this->dof_data[dataset]->get_function_values (fe_patch_values,
+                                                                 data.patch_values_system);
                        
+                   for (unsigned int component=0; component<data.n_components;
+                        ++component)
+                     {
                        switch (DH::dimension)
                          {
                            case 1:
                                  for (unsigned int x=0; x<n_points; ++x)
                                    for (unsigned int y=0; y<n_points; ++y)
-                                     patch->data(offset,
+                                     patches[angle].data(offset+component,
                                                  x*n_points + y)
-                                       = data.patch_values[x];
+                                       = data.patch_values_system[x](component);
                                  break;
-                                 
+                                     
                            case 2:
                                  for (unsigned int x=0; x<n_points; ++x)
                                    for (unsigned int y=0; y<n_points; ++y)
                                      for (unsigned int z=0; z<n_points; ++z)
-                                       patch->data(offset,
+                                       patches[angle].data(offset+component,
                                                    x*n_points*n_points +
-                                                   y +
-                                                   z*n_points)
-                                         = data.patch_values[x*n_points+z];
+                                                   y*n_points +
+                                                   z)
+                                         = data.patch_values_system[x*n_points+z](component);
                                  break;
-                                 
+                                     
                            default:
                                  Assert (false, ExcNotImplemented());
                          }
                      }
-                   else
-                                                      // system of components
-                     {
-                       this->dof_data[dataset]->get_function_values (fe_patch_values,
-                                                                     data.patch_values_system);
-                       
-                       for (unsigned int component=0; component<data.n_components;
-                            ++component)
-                         {
-                           switch (DH::dimension)
-                             {
-                               case 1:
-                                     for (unsigned int x=0; x<n_points; ++x)
-                                       for (unsigned int y=0; y<n_points; ++y)
-                                         patch->data(offset+component,
-                                                     x*n_points + y)
-                                           = data.patch_values_system[x](component);
-                                     break;
-                                     
-                               case 2:
-                                     for (unsigned int x=0; x<n_points; ++x)
-                                       for (unsigned int y=0; y<n_points; ++y)
-                                         for (unsigned int z=0; z<n_points; ++z)
-                                           patch->data(offset+component,
-                                                       x*n_points*n_points +
-                                                       y*n_points +
-                                                       z)
-                                             = data.patch_values_system[x*n_points+z](component);
-                                     break;
-                                     
-                               default:
-                                     Assert (false, ExcNotImplemented());
-                             }
-                         }
-                     }
-                 offset+=this->dof_data[dataset]->n_output_variables;
-               }
+                 }
+             offset+=this->dof_data[dataset]->n_output_variables;
+           }
                  
-                                              // then do the cell data
-             for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
+                                          // then do the cell data
+         for (unsigned int dataset=0; dataset<this->cell_data.size(); ++dataset)
+           {
+                                              // we need to get at
+                                              // the number of the
+                                              // cell to which this
+                                              // face belongs in
+                                              // order to access the
+                                              // cell data. this is
+                                              // not readily
+                                              // available, so choose
+                                              // the following rather
+                                              // inefficient way:
+             Assert ((*cell)->active(),
+                     ExcMessage("Cell must be active for cell data"));
+             const unsigned int cell_number
+               = std::distance (this->dofs->begin_active(),
+                                typename DH::active_cell_iterator(*cell));
+             const double value
+               = this->cell_data[dataset]->get_cell_data_value (cell_number);
+             switch (DH::dimension)
                {
-                 const double value
-                    = this->cell_data[dataset]->get_cell_data_value (cell_number);
-                 switch (DH::dimension)
-                   {
-                     case 1:
-                           for (unsigned int x=0; x<n_points; ++x)
-                             for (unsigned int y=0; y<n_points; ++y)
-                               patch->data(dataset+offset,
-                                           x*n_points +
-                                           y)
-                                 = value;
-                           break;
+                 case 1:
+                       for (unsigned int x=0; x<n_points; ++x)
+                         for (unsigned int y=0; y<n_points; ++y)
+                           patches[angle].data(dataset+offset,
+                                       x*n_points +
+                                       y)
+                             = value;
+                       break;
                            
-                     case 2:
-                           for (unsigned int x=0; x<n_points; ++x)
-                             for (unsigned int y=0; y<n_points; ++y)
-                               for (unsigned int z=0; z<n_points; ++z)
-                                 patch->data(dataset+offset,
-                                             x*n_points*n_points +
-                                             y*n_points +
-                                             z)
-                                   = value;
-                           break;
-
-                     default:
-                           Assert (false, ExcNotImplemented());
-                   }
+                 case 2:
+                       for (unsigned int x=0; x<n_points; ++x)
+                         for (unsigned int y=0; y<n_points; ++y)
+                           for (unsigned int z=0; z<n_points; ++z)
+                             patches[angle].data(dataset+offset,
+                                         x*n_points*n_points +
+                                         y*n_points +
+                                         z)
+                               = value;
+                       break;
+
+                 default:
+                       Assert (false, ExcNotImplemented());
                }
            }
-       }
-      
-                                      // next cell (patch) in this
-                                      // thread. note that we have
-                                      // already advanced the patches
-                                      // for the present cell,
-                                      // i.e. we only have to skip
-                                      // the cells belonging to other
-                                      // threads, not the ones
-                                      // belonging to this thread.
-      const int skip_threads = static_cast<signed int>(data.n_threads)-1;
-      for (int i=0; (i<skip_threads) && (cell != this->dofs->end()); ++i)
-       std::advance (patch, n_patches_per_circle);
-
-                                      // however, cell and cell
-                                      // number have not yet been
-                                      // increased
-      for (unsigned int i=0; (i<data.n_threads) && (cell != this->dofs->end()); ++i)
-       {
-         ++cell_number;
-         cell=next_cell(cell);
        }
     }
 }
@@ -406,7 +407,11 @@ void DataOutRotation<dim,DH>::build_some_patches (internal::DataOut::ParallelDat
 #else
 
 template <int dim, class DH>
-void DataOutRotation<dim,DH>::build_some_patches (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &)
+void
+DataOutRotation<dim,DH>::
+build_one_patch (const cell_iterator *,
+                internal::DataOutRotation::ParallelData<DH::dimension, DH::space_dimension> &,
+                std::vector<DataOutBase::Patch<DH::dimension+1,DH::space_dimension+1> > &)
 {
                                   // would this function make any
                                   // sense after all? who would want
@@ -420,118 +425,89 @@ void DataOutRotation<dim,DH>::build_some_patches (internal::DataOut::ParallelDat
 
 
 template <int dim, class DH>
-void DataOutRotation<dim,DH>::build_patches (
-  const unsigned int n_patches_per_circle,
-  const unsigned int nnnn_subdivisions,
-  const unsigned int n_threads_) 
+void DataOutRotation<dim,DH>::build_patches (const unsigned int n_patches_per_circle,
+                                            const unsigned int nnnn_subdivisions) 
 {
                                   // Check consistency of redundant
                                   // template parameter
   Assert (dim==DH::dimension, ExcDimensionMismatch(dim, DH::dimension));
-  
-  unsigned int n_subdivisions = (nnnn_subdivisions != 0)
-                               ? nnnn_subdivisions
-                               : this->default_subdivisions;
-  
-  Assert (n_subdivisions >= 1,
-         ExcInvalidNumberOfSubdivisions(n_subdivisions));
-
   typedef DataOut_DoFData<DH,DH::dimension+1> BaseClass;
   Assert (this->dofs != 0, typename BaseClass::ExcNoDoFHandlerSelected());
 
-  Assert (!DEAL_II_USE_MT || (n_threads_ >= 1),
-         ExcMessage ("Must run with at least one thread!"));
-  const unsigned int n_threads = (DEAL_II_USE_MT ? n_threads_ : 1);
-
-                                  // before we start the loop:
-                                  // create a quadrature rule that
-                                  // actually has the points on this
-                                  // patch
-  QTrapez<1>     q_trapez;
-  QIterated<DH::dimension> patch_points (q_trapez, n_subdivisions);
+  const unsigned int n_subdivisions = (nnnn_subdivisions != 0)
+                                     ? nnnn_subdivisions
+                                     : this->default_subdivisions;  
+  Assert (n_subdivisions >= 1,
+         ExcInvalidNumberOfSubdivisions(n_subdivisions));
 
-  const unsigned int n_q_points     = patch_points.n_quadrature_points;
+  const QTrapez<1>     q_trapez;
+  const QIterated<DH::dimension> patch_points (q_trapez, n_subdivisions);
+  
   const unsigned int n_components   = this->dofs->get_fe().n_components();
   unsigned int n_datasets=this->cell_data.size();
   for (unsigned int i=0; i<this->dof_data.size(); ++i)
     n_datasets+= this->dof_data[i]->n_output_variables;
-  
-                                  // clear the patches array
-  if (true)
-    {
-      std::vector< dealii::DataOutBase::Patch<DH::dimension+1> > dummy;
-      this->patches.swap (dummy);
-    };
+
+  UpdateFlags update_flags=update_values | update_quadrature_points;
+  for (unsigned int i=0; i<this->dof_data.size(); ++i)
+    if (this->dof_data[i]->postprocessor)
+      update_flags |= this->dof_data[i]->postprocessor->get_needed_update_flags();
+                                  // perhaps update_normal_vectors is present,
+                                  // which would only be useful on faces, but
+                                  // we may not use it here.
+  Assert (!(update_flags & update_normal_vectors),
+         ExcMessage("The update of normal vectors may not be requested for "
+                    "evaluation of data on cells via DataPostprocessor."));
   
                                   // first count the cells we want to
                                   // create patches of and make sure
                                   // there is enough memory for that
-  unsigned int n_patches = 0;
+  std::vector<cell_iterator> all_cells;
   for (cell_iterator cell=first_cell(); cell != this->dofs->end();
        cell = next_cell(cell))
-    ++n_patches;
+    all_cells.push_back (cell);
+  
                                   // then also take into account that
                                   // we want more than one patch to
                                   // come out of every cell, as they
                                   // are repeated around the axis of
                                   // rotation
-  n_patches *= n_patches_per_circle;
-
-  std::vector<internal::DataOut::ParallelData<DH::dimension, DH::dimension> > thread_data(n_threads);
-
-                                  // init data for the threads
-  for (unsigned int i=0;i<n_threads;++i)
+  this->patches.clear();
+  this->patches.reserve (all_cells.size() * n_patches_per_circle);
+  
+  
+  std::vector<unsigned int> n_postprocessor_outputs (this->dof_data.size());
+  for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
+    if (this->dof_data[dataset]->postprocessor)
+      n_postprocessor_outputs[dataset] = this->dof_data[dataset]->n_output_variables;
+    else
+      n_postprocessor_outputs[dataset] = 0;
+
+  internal::DataOutRotation::ParallelData<DH::dimension, DH::space_dimension>
+    thread_data (patch_points, n_components, n_datasets,
+                n_subdivisions, n_patches_per_circle,
+                n_postprocessor_outputs, this->dofs->get_fe(),
+                update_flags);
+  std::vector<DataOutBase::Patch<DH::dimension+1,DH::space_dimension+1> >
+    new_patches (n_patches_per_circle);
+  for (unsigned int i=0; i<new_patches.size(); ++i)
     {
-      thread_data[i].n_threads            = n_threads;
-      thread_data[i].this_thread          = i;
-      thread_data[i].n_components         = n_components;
-      thread_data[i].n_datasets           = n_datasets;
-      thread_data[i].n_patches_per_circle = n_patches_per_circle;
-      thread_data[i].n_subdivisions       = n_subdivisions;
-      thread_data[i].patch_values.resize (n_q_points);
-      thread_data[i].patch_values_system.resize (n_q_points);
-      thread_data[i].patch_gradients.resize (n_q_points);
-      thread_data[i].patch_gradients_system.resize (n_q_points);
-      thread_data[i].patch_hessians.resize (n_q_points);
-      thread_data[i].patch_hessians_system.resize (n_q_points);
-      thread_data[i].dummy_normals.clear();
-      thread_data[i].postprocessed_values.resize(this->dof_data.size());
-      
-      for (unsigned int k=0; k<n_q_points; ++k)
-       {
-         thread_data[i].patch_values_system[k].reinit(n_components);
-         thread_data[i].patch_gradients_system[k].resize(n_components);
-         thread_data[i].patch_hessians_system[k].resize(n_components);
-       }
-
-      for (unsigned int dataset=0; dataset<this->dof_data.size(); ++dataset)
-       if (this->dof_data[dataset]->postprocessor)
-         thread_data[i].postprocessed_values[dataset].resize(n_q_points,Vector<double>(this->dof_data[dataset]->n_output_variables));
+      new_patches[i].n_subdivisions = n_subdivisions;
+      new_patches[i].data.reinit (n_datasets,
+                                 patch_points.n_quadrature_points
+                                 * (n_subdivisions+1));
     }
 
-                                  // create the patches with default
-                                  // values. note that the evaluation
-                                  // points on the cell have to be
-                                  // repeated in angular direction
-  dealii::DataOutBase::Patch<DH::dimension+1>  default_patch;
-  default_patch.n_subdivisions = n_subdivisions;
-  default_patch.data.reinit (n_datasets,
-                            n_q_points * (n_subdivisions+1));
-  this->patches.insert (this->patches.end(), n_patches, default_patch);
-
-  if (DEAL_II_USE_MT)
-    {
-      void (DataOutRotation<dim,DH>::*p) (internal::DataOut::ParallelData<DH::dimension, DH::dimension> &)
-        = &DataOutRotation<dim,DH>::build_some_patches;
-
-      Threads::ThreadGroup<> threads;
-      for (unsigned int l=0;l<n_threads;++l)
-        threads += Threads::spawn (*this, p) (thread_data[l]);
-      threads.join_all();
-    }
-  else
-                                     // just one thread
-    build_some_patches(thread_data[0]);
+                                  // now build the patches in parallel
+  WorkStream::run (&all_cells[0],
+                  &all_cells[0]+all_cells.size(),
+                  std_cxx1x::bind(&DataOutRotation<dim,DH>::build_one_patch,
+                                  *this, _1, _2, _3),
+                  std_cxx1x::bind(&internal::DataOutRotation
+                                  ::append_patch_to_list<dim,DH::space_dimension>,
+                                  _1, std_cxx1x::ref(this->patches)),
+                  thread_data,
+                  new_patches);
 }
 
 
index 3cae0d07052254cc046c9dd714f544cb330f61c4..8aeaaa50b0ba68ef17728c8b46300c18f2d9d224 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -636,8 +636,6 @@ approximate_derivative (const Mapping<dim,spacedim>    &mapping,
     = Threads::split_interval (0, dof_handler.get_tria().n_active_cells(),
                               n_threads);
 
-  Threads::ThreadGroup<> threads;
-  
   typedef void (*FunPtr) (const Mapping<dim,spacedim>    &,
                           const DH<dim,spacedim>         &,
                           const InputVector     &,
@@ -647,13 +645,16 @@ approximate_derivative (const Mapping<dim,spacedim>    &mapping,
   FunPtr fun_ptr = &DerivativeApproximation::
                   template approximate<DerivativeDescription,dim,
                   DH,InputVector>;
-  
+
+//TODO: Use WorkStream here  
+  Threads::TaskGroup<> tasks;  
   for (unsigned int i=0; i<n_threads; ++i)
-    threads += Threads::spawn (fun_ptr)
-              (mapping, dof_handler, solution, component,
-               index_intervals[i],
-               derivative_norm);
-  threads.join_all ();
+    tasks += Threads::new_task (fun_ptr,
+                               mapping, dof_handler,
+                               solution, component,
+                               index_intervals[i],
+                               derivative_norm);
+  tasks.join_all ();
 }
 
 
index 15a88199a669dc4ff50d80496a10c0038ac4a792..7ad59ded8f196f44af4031ee59def8b4cb0577c4 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -15,6 +15,7 @@
 #include <base/thread_management.h>
 #include <base/quadrature.h>
 #include <base/quadrature_lib.h>
+#include <base/work_stream.h>
 #include <lac/vector.h>
 #include <lac/block_vector.h>
 #include <lac/petsc_vector.h>
@@ -34,6 +35,8 @@
 #include <hp/mapping_collection.h>
 #include <numerics/error_estimator.h>
 
+#include <base/std_cxx1x/bind.h>
+
 #include <numeric>
 #include <algorithm>
 #include <cmath>
 DEAL_II_NAMESPACE_OPEN
 
 
-
-static
-inline
-double sqr (const double x)
+namespace
 {
-  return x*x;
+  inline
+  double sqr (const double x)
+  {
+    return x*x;
+  }
+
+
+  template <typename CellIterator>
+  inline
+  void advance_by_n (CellIterator &cell,
+                    const unsigned int n)
+  {
+                                    // store a pointer to the end
+                                    // iterator, since we can't get at
+                                    // it any more once cell is already
+                                    // the end iterator (in that case
+                                    // dereferencing cell-> triggers an
+                                    // assertion)
+    const CellIterator endc = cell->get_dof_handler().end();
+    for (unsigned int t=0; ((t<n) && (cell!=endc)); ++t, ++cell)
+      ;
+  }
 }
 
 
-template <typename CellIterator>
-inline
-void advance_by_n (CellIterator &cell,
-                   const unsigned int n)
+namespace internal
 {
-                                  // store a pointer to the end
-                                  // iterator, since we can't get at
-                                  // it any more once cell is already
-                                  // the end iterator (in that case
-                                  // dereferencing cell-> triggers an
-                                  // assertion)
-  const CellIterator endc = cell->get_dof_handler().end();
-  for (unsigned int t=0; ((t<n) && (cell!=endc)); ++t, ++cell)
-    ;
+  namespace 
+  {
+                                    /**
+                                     * All small temporary data
+                                     * objects that are needed once
+                                     * per thread by the several
+                                     * functions of the error
+                                     * estimator are gathered in this
+                                     * struct. The reason for this
+                                     * structure is mainly that we
+                                     * have a number of functions
+                                     * that operate on cells or faces
+                                     * and need a number of small
+                                     * temporary data objects. Since
+                                     * these functions may run in
+                                     * parallel, we cannot make these
+                                     * objects member variables of
+                                     * the enclosing class. On the
+                                     * other hand, declaring them
+                                     * locally in each of these
+                                     * functions would require their
+                                     * reallocating every time we
+                                     * visit the next cell or face,
+                                     * which we found can take a
+                                     * significant amount of time if
+                                     * it happens often even in the
+                                     * single threaded case (10-20
+                                     * per cent in our measurements);
+                                     * however, most importantly,
+                                     * memory allocation requires
+                                     * synchronisation in
+                                     * multithreaded mode. While that
+                                     * is done by the C++ library and
+                                     * has not to be handcoded, it
+                                     * nevertheless seriously damages
+                                     * the ability to efficiently run
+                                     * the functions of this class in
+                                     * parallel, since they are quite
+                                     * often blocked by these
+                                     * synchronisation points,
+                                     * slowing everything down by a
+                                     * factor of two or three.
+                                     *
+                                     * Thus, every thread gets an
+                                     * instance of this class to work
+                                     * with and needs not allocate
+                                     * memory itself, or synchronise
+                                     * with other threads.
+                                     *
+                                     * The sizes of the arrays are
+                                     * initialized with the maximal number of
+                                     * entries necessary for the hp
+                                     * case. Within the loop over individual
+                                     * cells, we then resize the arrays as
+                                     * necessary. Since for std::vector
+                                     * resizing to a smaller size doesn't
+                                     * imply memory allocation, this is fast.
+                                     */
+    template <class DH>
+    struct ParallelData
+    {
+       static const unsigned int dim      = DH::dimension;
+       static const unsigned int spacedim = DH::space_dimension;
+       
+                                        /**
+                                         * The finite element to be used.
+                                         */
+       const dealii::hp::FECollection<dim,spacedim> finite_element;
+
+                                        /**
+                                         * The quadrature formulas to be used for
+                                         * the faces.
+                                         */
+       const dealii::hp::QCollection<dim-1> face_quadratures;
+
+                                        /**
+                                         * FEFaceValues objects to integrate over
+                                         * the faces of the current and
+                                         * potentially of neighbor cells.
+                                         */
+       dealii::hp::FEFaceValues<dim>    fe_face_values_cell;
+       dealii::hp::FEFaceValues<dim>    fe_face_values_neighbor;
+       dealii::hp::FESubfaceValues<dim> fe_subface_values;
+    
+                                        /**
+                                         * A vector to store the jump
+                                         * of the normal vectors in
+                                         * the quadrature points for
+                                         * each of the solution
+                                         * vectors (i.e. a temporary
+                                         * value). This vector is not
+                                         * allocated inside the
+                                         * functions that use it, but
+                                         * rather globally, since
+                                         * memory allocation is slow,
+                                         * in particular in presence
+                                         * of multiple threads where
+                                         * synchronisation makes
+                                         * things even slower.
+                                         */ 
+       std::vector<std::vector<std::vector<double> > > phi;
+
+                                        /**
+                                         * A vector for the gradients of
+                                         * the finite element function
+                                         * on one cell
+                                         *
+                                         * Let psi be a short name
+                                         * for <tt>a grad u_h</tt>, where
+                                         * the third index be the
+                                         * component of the finite
+                                         * element, and the second
+                                         * index the number of the
+                                         * quadrature point. The
+                                         * first index denotes the
+                                         * index of the solution
+                                         * vector.
+                                         */
+       std::vector<std::vector<std::vector<Tensor<1,dim> > > > psi;
+
+                                        /**
+                                         * The same vector for a neighbor cell
+                                         */
+       std::vector<std::vector<std::vector<Tensor<1,dim> > > > neighbor_psi;
+
+                                        /**
+                                         * The normal vectors of the finite
+                                         * element function on one face
+                                         */
+       std::vector<Point<dim> > normal_vectors;
+
+                                        /**
+                                         * Two arrays needed for the
+                                         * values of coefficients in
+                                         * the jumps, if they are
+                                         * given.
+                                         */
+       std::vector<double>                  coefficient_values1;
+       std::vector<dealii::Vector<double> > coefficient_values;
+
+                                        /**
+                                         * Array for the products of
+                                         * Jacobian determinants and
+                                         * weights of quadraturs
+                                         * points.
+                                         */
+       std::vector<double>          JxW_values;
+    
+                                        /**
+                                         * The subdomain id we are to care
+                                         * for.
+                                         */
+       const unsigned int subdomain_id;
+                                        /**
+                                         * The material id we are to care
+                                         * for.
+                                         */
+       const unsigned int material_id;
+
+                                        /**
+                                         * Some more references to input data to
+                                         * the KellyErrorEstimator::estimate()
+                                         * function.
+                                         */
+       const typename FunctionMap<dim>::type *neumann_bc;
+       const std::vector<bool>                component_mask;
+       const Function<dim>                   *coefficients;
+    
+                                        /**
+                                         * Constructor.
+                                         */
+       template <class FE>
+       ParallelData (const FE                                           &fe,
+                     const dealii::hp::QCollection<dim-1>               &face_quadratures,
+                     const dealii::hp::MappingCollection<dim, spacedim> &mapping,
+                     const bool         need_quadrature_points,
+                     const unsigned int n_solution_vectors,
+                     const unsigned int subdomain_id,
+                     const unsigned int material_id,
+                     const typename FunctionMap<dim>::type *neumann_bc,
+                     const std::vector<bool>                component_mask,
+                     const Function<dim>                   *coefficients);
+
+                                        /**
+                                         * Resize the arrays so that they fit the
+                                         * number of quadrature points associated
+                                         * with the given finite element index
+                                         * into the hp collections.
+                                         */
+       void resize (const unsigned int active_fe_index);
+    };
+
+
+    template <class DH>
+    template <class FE>
+    ParallelData<DH>::
+    ParallelData (const FE                                           &fe,
+                 const dealii::hp::QCollection<dim-1>               &face_quadratures,
+                 const dealii::hp::MappingCollection<dim, spacedim> &mapping,
+                 const bool     need_quadrature_points,
+                 const unsigned int n_solution_vectors,
+                 const unsigned int subdomain_id,
+                 const unsigned int material_id,
+                 const typename FunctionMap<dim>::type *neumann_bc,
+                 const std::vector<bool>                component_mask,
+                 const Function<dim>                   *coefficients)
+                   :
+                   finite_element (fe),
+                   face_quadratures (face_quadratures),
+                   fe_face_values_cell (mapping,
+                                        finite_element,
+                                        face_quadratures,
+                                        update_gradients      |
+                                        update_JxW_values     |
+                                        (need_quadrature_points  ?
+                                         update_quadrature_points :
+                                         UpdateFlags()) |
+                                        update_normal_vectors),
+                   fe_face_values_neighbor (mapping,
+                                            finite_element,
+                                            face_quadratures,
+                                            update_gradients),
+                   fe_subface_values (mapping,
+                                      finite_element,
+                                      face_quadratures,
+                                      update_gradients),
+                   phi (n_solution_vectors,
+                        std::vector<std::vector<double> >
+                        (face_quadratures.max_n_quadrature_points(),
+                         std::vector<double> (fe.n_components()))),
+                   psi (n_solution_vectors,
+                        std::vector<std::vector<Tensor<1,dim> > >
+                        (face_quadratures.max_n_quadrature_points(),
+                         std::vector<Tensor<1,dim> > (fe.n_components()))),
+                   neighbor_psi (n_solution_vectors,
+                                 std::vector<std::vector<Tensor<1,dim> > >
+                                 (face_quadratures.max_n_quadrature_points(),
+                                  std::vector<Tensor<1,dim> > (fe.n_components()))),
+                   normal_vectors (face_quadratures.max_n_quadrature_points()),
+                   coefficient_values1 (face_quadratures.max_n_quadrature_points()),
+                   coefficient_values (face_quadratures.max_n_quadrature_points(),
+                                       dealii::Vector<double> (fe.n_components())),
+                   JxW_values (face_quadratures.max_n_quadrature_points()),
+                   subdomain_id (subdomain_id),
+                   material_id (material_id),
+                   neumann_bc (neumann_bc),
+                   component_mask (component_mask),
+                   coefficients (coefficients)
+    {}
+
+
+
+    template <class DH>
+    void
+    ParallelData<DH>::resize (const unsigned int active_fe_index)
+    {
+      const unsigned int n_q_points   = face_quadratures[active_fe_index].n_quadrature_points;
+      const unsigned int n_components = finite_element.n_components();
+  
+      normal_vectors.resize(n_q_points);
+      coefficient_values1.resize(n_q_points);
+      coefficient_values.resize(n_q_points);
+      JxW_values.resize(n_q_points);  
+
+      for (unsigned int i=0; i<phi.size(); ++i)
+       {
+         phi[i].resize(n_q_points);
+         psi[i].resize(n_q_points);
+         neighbor_psi[i].resize(n_q_points);
+
+         for (unsigned int qp=0;qp<n_q_points;++qp)
+           {
+             phi[i][qp].resize(n_components);
+             psi[i][qp].resize(n_components);
+             neighbor_psi[i][qp].resize(n_components);
+           }
+       }
+
+      for (unsigned int qp=0; qp<n_q_points; ++qp)
+       coefficient_values[qp].reinit(n_components);
+    }
+
+
+
+                                    /**
+                                     * Copy data from the
+                                     * local_face_integrals map of a single
+                                     * ParallelData object into a global such
+                                     * map. This is the copier stage of a
+                                     * WorkStream pipeline.
+                                     */
+    template <class DH>
+    void
+    copy_local_to_global (const        std::map<typename DH::face_iterator,std::vector<double> > &local_face_integrals,
+                         std::map<typename DH::face_iterator,std::vector<double> > &face_integrals)
+    {
+  
+                                      // now copy locally computed elements
+                                      // into the global map
+      for (typename std::map<typename DH::face_iterator,std::vector<double> >::const_iterator
+            p=local_face_integrals.begin();
+          p!=local_face_integrals.end();
+          ++p)
+       {
+                                          // double check that the
+                                          // element does not already
+                                          // exists in the global map
+         Assert (face_integrals.find (p->first) == face_integrals.end(),
+                 ExcInternalError());
+         for (unsigned int i=0; i<p->second.size(); ++i)
+           Assert (p->second[i] >= 0, ExcInternalError());
+      
+         face_integrals[p->first] = p->second;
+       }
+    }
+
+
+                                    /**
+                                     * Actually do the computation on
+                                     * a face which has no hanging
+                                     * nodes (it is regular), i.e.
+                                     * either on the other side there
+                                     * is nirvana (face is at
+                                     * boundary), or the other side's
+                                     * refinement level is the same
+                                     * as that of this side, then
+                                     * handle the integration of
+                                     * these both cases together.
+                                     */    
+    template <typename InputVector, class DH>
+    void 
+    integrate_over_regular_face (const std::vector<const InputVector*>   &solutions,
+                                ParallelData<DH>                        &parallel_data,
+                                std::map<typename DH::face_iterator,std::vector<double> > &local_face_integrals,
+                                const typename DH::active_cell_iterator &cell,
+                                const unsigned int                       face_no,
+                                dealii::hp::FEFaceValues<DH::dimension> &fe_face_values_cell,
+                                dealii::hp::FEFaceValues<DH::dimension> &fe_face_values_neighbor)
+    {
+      const unsigned int dim = DH::dimension;
+      
+      const typename DH::face_iterator face = cell->face(face_no);
+      const unsigned int n_q_points         = parallel_data.face_quadratures[cell->active_fe_index()]
+                                             .n_quadrature_points,
+                        n_components       = parallel_data.finite_element.n_components(),
+                        n_solution_vectors = solutions.size();
+  
+  
+                                      // initialize data of the restriction
+                                      // of this cell to the present face
+      fe_face_values_cell.reinit (cell, face_no,
+                                 cell->active_fe_index());
+  
+                                      // get gradients of the finite element
+                                      // function on this cell
+      for (unsigned int n=0; n<n_solution_vectors; ++n)
+       fe_face_values_cell.get_present_fe_values()
+         .get_function_grads (*solutions[n], parallel_data.psi[n]);
+  
+                                      // now compute over the other side of
+                                      // the face
+      if (face->at_boundary() == false)
+                                        // internal face; integrate jump
+                                        // of gradient across this face
+       {
+         Assert (cell->neighbor(face_no).state() == IteratorState::valid,
+                 ExcInternalError());      
+      
+         const typename DH::active_cell_iterator neighbor = cell->neighbor(face_no);
+      
+                                          // find which number the
+                                          // current face has relative to
+                                          // the neighboring cell
+         const unsigned int neighbor_neighbor
+           = cell->neighbor_of_neighbor (face_no);
+         Assert (neighbor_neighbor<GeometryInfo<dim>::faces_per_cell,
+                 ExcInternalError());
+      
+                                          // get restriction of finite element
+                                          // function of @p{neighbor} to the
+                                          // common face. in the hp case, use the
+                                          // quadrature formula that matches the
+                                          // one we would use for the present
+                                          // cell
+         fe_face_values_neighbor.reinit (neighbor, neighbor_neighbor,
+                                         cell->active_fe_index());
+      
+                                          // get gradients on neighbor cell
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           {
+             fe_face_values_neighbor.get_present_fe_values()
+               .get_function_grads (*solutions[n],
+                                    parallel_data.neighbor_psi[n]);
+      
+                                              // compute the jump in the gradients
+             for (unsigned int component=0; component<n_components; ++component)
+               for (unsigned int p=0; p<n_q_points; ++p)
+                 parallel_data.psi[n][p][component] -= parallel_data.neighbor_psi[n][p][component];
+           }
+       }
+
+
+                                      // now psi contains the following:
+                                      // - for an internal face, psi=[grad u]
+                                      // - for a neumann boundary face,
+                                      //   psi=grad u
+                                      // each component being the
+                                      // mentioned value at one of the
+                                      // quadrature points
+  
+                                      // next we have to multiply this with
+                                      // the normal vector. Since we have
+                                      // taken the difference of gradients
+                                      // for internal faces, we may chose
+                                      // the normal vector of one cell,
+                                      // taking that of the neighbor
+                                      // would only change the sign. We take
+                                      // the outward normal.
+  
+      parallel_data.normal_vectors =
+       fe_face_values_cell.get_present_fe_values().get_normal_vectors();
+  
+      for (unsigned int n=0; n<n_solution_vectors; ++n)
+       for (unsigned int component=0; component<n_components; ++component)
+         for (unsigned int point=0; point<n_q_points; ++point)
+           parallel_data.phi[n][point][component]
+             = (parallel_data.psi[n][point][component] *
+                parallel_data.normal_vectors[point]);
+  
+                                      // if a coefficient was given: use that
+                                      // to scale the jump in the gradient
+      if (parallel_data.coefficients != 0)
+       {
+                                          // scalar coefficient
+         if (parallel_data.coefficients->n_components == 1)
+           {
+         
+             parallel_data.coefficients
+               ->value_list (fe_face_values_cell.get_present_fe_values()
+                             .get_quadrature_points(),
+                             parallel_data.coefficient_values1);
+             for (unsigned int n=0; n<n_solution_vectors; ++n)
+               for (unsigned int component=0; component<n_components; ++component)
+                 for (unsigned int point=0; point<n_q_points; ++point)
+                   parallel_data.phi[n][point][component] *=
+                     parallel_data.coefficient_values1[point];
+           }
+         else
+                                            // vector-valued coefficient
+           {
+             parallel_data.coefficients
+               ->vector_value_list (fe_face_values_cell.get_present_fe_values()
+                                    .get_quadrature_points(),
+                                    parallel_data.coefficient_values);
+             for (unsigned int n=0; n<n_solution_vectors; ++n)
+               for (unsigned int component=0; component<n_components; ++component)
+                 for (unsigned int point=0; point<n_q_points; ++point)
+                   parallel_data.phi[n][point][component] *=
+                     parallel_data.coefficient_values[point](component);
+           }
+       }
+
+
+      if (face->at_boundary() == true)
+                                        // neumann boundary face. compute
+                                        // difference between normal
+                                        // derivative and boundary function
+       {
+         const unsigned char boundary_indicator = face->boundary_indicator();
+      
+         Assert (parallel_data.neumann_bc->find(boundary_indicator) !=
+                 parallel_data.neumann_bc->end(),
+                 ExcInternalError ());
+                                          // get the values of the boundary
+                                          // function at the quadrature
+                                          // points
+         if (n_components == 1)
+           {
+             std::vector<double> g(n_q_points);
+             parallel_data.neumann_bc->find(boundary_indicator)->second
+               ->value_list (fe_face_values_cell.get_present_fe_values()
+                             .get_quadrature_points(), g);
+         
+             for (unsigned int n=0; n<n_solution_vectors; ++n)
+               for (unsigned int point=0; point<n_q_points; ++point)
+                 parallel_data.phi[n][point][0] -= g[point];
+           }
+         else
+           {
+             std::vector<dealii::Vector<double> >
+               g(n_q_points, dealii::Vector<double>(n_components));
+             parallel_data.neumann_bc->find(boundary_indicator)->second
+               ->vector_value_list (fe_face_values_cell.get_present_fe_values()
+                                    .get_quadrature_points(),
+                                    g);
+         
+             for (unsigned int n=0; n<n_solution_vectors; ++n)
+               for (unsigned int component=0; component<n_components; ++component)
+                 for (unsigned int point=0; point<n_q_points; ++point)
+                   parallel_data.phi[n][point][component] -= g[point](component);
+           }
+       }
+
+
+                                      // now phi contains the following:
+                                      // - for an internal face, phi=[a du/dn]
+                                      // - for a neumann boundary face,
+                                      //   phi=a du/dn-g
+                                      // each component being the
+                                      // mentioned value at one of the
+                                      // quadrature points
+
+      parallel_data.JxW_values
+       = fe_face_values_cell.get_present_fe_values().get_JxW_values();
+  
+                                      // take the square of the phi[i]
+                                      // for integration, and sum up
+      std::vector<double> face_integral (n_solution_vectors, 0);
+      for (unsigned int n=0; n<n_solution_vectors; ++n)
+       for (unsigned int component=0; component<n_components; ++component)
+         if (parallel_data.component_mask[component] == true)
+           for (unsigned int p=0; p<n_q_points; ++p)
+             face_integral[n] += dealii::sqr(parallel_data.phi[n][p][component]) *
+                                 parallel_data.JxW_values[p];
+
+      local_face_integrals[face] = face_integral;
+    }
+
+
+
+
+                                    /**
+                                     * The same applies as for the
+                                     * function above, except that
+                                     * integration is over face
+                                     * @p face_no of @p cell, where
+                                     * the respective neighbor is
+                                     * refined, so that the
+                                     * integration is a bit more
+                                     * complex.
+                                     */
+    template <typename InputVector, class DH>
+    void
+    integrate_over_irregular_face (const std::vector<const InputVector*>      &solutions,
+                                  ParallelData<DH>                           &parallel_data,
+                                  std::map<typename DH::face_iterator,std::vector<double> > &local_face_integrals,
+                                  const typename DH::active_cell_iterator    &cell,
+                                  const unsigned int                          face_no,
+                                  dealii::hp::FEFaceValues<DH::dimension>    &fe_face_values,
+                                  dealii::hp::FESubfaceValues<DH::dimension> &fe_subface_values)
+    {
+      const unsigned int dim = DH::dimension;
+      
+      const typename DH::cell_iterator neighbor = cell->neighbor(face_no);
+      const unsigned int n_q_points         = parallel_data.face_quadratures[cell->active_fe_index()]
+                                             .n_quadrature_points,
+                        n_components       = parallel_data.finite_element.n_components(),
+                        n_solution_vectors = solutions.size();
+      const typename DH::face_iterator
+       face=cell->face(face_no);
+
+      Assert (neighbor.state() == IteratorState::valid, ExcInternalError());
+      Assert (face->has_children(), ExcInternalError());
+                                      // set up a vector of the gradients
+                                      // of the finite element function
+                                      // on this cell at the quadrature
+                                      // points
+                                      //
+                                      // let psi be a short name for
+                                      // [a grad u_h], where the second
+                                      // index be the component of the
+                                      // finite element, and the first
+                                      // index the number of the
+                                      // quadrature point
+  
+                                      // store which number @p{cell} has
+                                      // in the list of neighbors of
+                                      // @p{neighbor}
+      const unsigned int neighbor_neighbor
+       = cell->neighbor_of_neighbor (face_no);
+      Assert (neighbor_neighbor<GeometryInfo<dim>::faces_per_cell,
+             ExcInternalError());
+  
+                                      // loop over all subfaces
+      for (unsigned int subface_no=0; subface_no<face->n_children(); ++subface_no)
+       {
+                                          // get an iterator pointing to the
+                                          // cell behind the present subface
+         const typename DH::active_cell_iterator neighbor_child
+           = cell->neighbor_child_on_subface (face_no, subface_no);
+         Assert (!neighbor_child->has_children(),
+                 ExcInternalError());
+      
+                                          // restrict the finite element
+                                          // on the present cell to the
+                                          // subface
+         fe_subface_values.reinit (cell, face_no, subface_no,
+                                   cell->active_fe_index());
+
+                                          // restrict the finite element
+                                          // on the neighbor cell to the
+                                          // common @p{subface}.
+         fe_face_values.reinit (neighbor_child, neighbor_neighbor,
+                                cell->active_fe_index());
+
+                                          // store the gradient of the
+                                          // solution in psi
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           fe_subface_values.get_present_fe_values()
+             .get_function_grads (*solutions[n], parallel_data.psi[n]);
+
+                                          // store the gradient from the
+                                          // neighbor's side in
+                                          // @p{neighbor_psi}
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           fe_face_values.get_present_fe_values()
+             .get_function_grads (*solutions[n], parallel_data.neighbor_psi[n]);
+      
+                                          // compute the jump in the gradients
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           for (unsigned int component=0; component<n_components; ++component)
+             for (unsigned int p=0; p<n_q_points; ++p)
+               parallel_data.psi[n][p][component] -=
+                 parallel_data.neighbor_psi[n][p][component];
+
+                                          // note that unlike for the
+                                          // case of regular faces
+                                          // (treated in the other
+                                          // function of this class), we
+                                          // have not to take care of
+                                          // boundary faces here, since
+                                          // they always are regular.
+      
+                                          // next we have to multiply this with
+                                          // the normal vector. Since we have
+                                          // taken the difference of gradients
+                                          // for internal faces, we may chose
+                                          // the normal vector of one cell,
+                                          // taking that of the neighbor
+                                          // would only change the sign. We take
+                                          // the outward normal.
+                                          //
+                                          // let phi be the name of the integrand
+     
+         parallel_data.normal_vectors
+           = fe_face_values.get_present_fe_values().get_normal_vectors();
+
+
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           for (unsigned int component=0; component<n_components; ++component)
+             for (unsigned int point=0; point<n_q_points; ++point)
+               parallel_data.phi[n][point][component] = (parallel_data.psi[n][point][component]*
+                                                         parallel_data.normal_vectors[point]);
+      
+                                          // if a coefficient was given: use that
+                                          // to scale the jump in the gradient
+         if (parallel_data.coefficients != 0)
+           {
+                                              // scalar coefficient
+             if (parallel_data.coefficients->n_components == 1)
+               {
+                 parallel_data.coefficients
+                   ->value_list (fe_face_values.get_present_fe_values()
+                                 .get_quadrature_points(),
+                                 parallel_data.coefficient_values1);
+                 for (unsigned int n=0; n<n_solution_vectors; ++n)
+                   for (unsigned int component=0; component<n_components; ++component)
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       parallel_data.phi[n][point][component] *=
+                         parallel_data.coefficient_values1[point];
+               }
+             else
+                                                // vector-valued coefficient
+               {
+                 parallel_data.coefficients
+                   ->vector_value_list (fe_face_values.get_present_fe_values()
+                                        .get_quadrature_points(),
+                                        parallel_data.coefficient_values);
+                 for (unsigned int n=0; n<n_solution_vectors; ++n)
+                   for (unsigned int component=0; component<n_components; ++component)
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       parallel_data.phi[n][point][component] *=
+                         parallel_data.coefficient_values[point](component);
+               }
+           }
+
+                                          // get the weights for the
+                                          // integration. note that it
+                                          // does not matter whether we
+                                          // take the JxW values from the
+                                          // fe_face_values or the
+                                          // fe_subface_values, as the
+                                          // first is on the small
+                                          // neighbor cell, while the
+                                          // latter is on the refined
+                                          // face of the big cell here
+         parallel_data.JxW_values
+           = fe_face_values.get_present_fe_values().get_JxW_values();
+      
+                                          // take the square of the phi[i]
+                                          // for integration, and sum up
+         std::vector<double> face_integral (n_solution_vectors, 0);
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           for (unsigned int component=0; component<n_components; ++component)
+             if (parallel_data.component_mask[component] == true)
+               for (unsigned int p=0; p<n_q_points; ++p)
+                 face_integral[n] += dealii::sqr(parallel_data.phi[n][p][component]) *
+                                     parallel_data.JxW_values[p];
+
+         local_face_integrals[neighbor_child->face(neighbor_neighbor)]
+           = face_integral;
+       }
+
+
+                                      // finally loop over all subfaces to
+                                      // collect the contributions of the
+                                      // subfaces and store them with the
+                                      // mother face
+      std::vector<double> sum (n_solution_vectors, 0);
+      for (unsigned int subface_no=0; subface_no<face->n_children(); ++subface_no) 
+       {
+         Assert (local_face_integrals.find(face->child(subface_no)) !=
+                 local_face_integrals.end(),
+                 ExcInternalError());
+         Assert (local_face_integrals[face->child(subface_no)][0] >= 0,
+                 ExcInternalError());
+      
+         for (unsigned int n=0; n<n_solution_vectors; ++n)
+           sum[n] += local_face_integrals[face->child(subface_no)][n];
+       }
+
+      local_face_integrals[face] = sum;
+    }
+
+
+                                    /**
+                                     * Computate the error on the faces of a
+                                     * single cell.
+                                     *
+                                     * This function is only needed
+                                     * in two or three dimensions.
+                                     * The error estimator in one
+                                     * dimension is implemented
+                                     * seperatly.
+                                     */    
+    template <int dim, int spacedim, typename InputVector, class DH>
+    void
+    estimate_one_cell (const typename DH::active_cell_iterator &cell,
+                      ParallelData<DH>                    &parallel_data,
+                      std::map<typename DH::face_iterator,std::vector<double> > &local_face_integrals,
+                      const std::vector<const InputVector*> &solutions)
+    {
+      const unsigned int n_solution_vectors = solutions.size();
+  
+      const unsigned int subdomain_id = parallel_data.subdomain_id;
+      const unsigned int material_id  = parallel_data.material_id;
+
+                                      // empty our own copy of the local face
+                                      // integrals
+      local_face_integrals.clear();
+      
+                                      // loop over all faces of this cell
+      for (unsigned int face_no=0;
+          face_no<GeometryInfo<dim>::faces_per_cell; ++face_no)
+       {
+         const typename DH::face_iterator
+           face=cell->face(face_no);
+
+                                          // make sure we do work
+                                          // only once: this face
+                                          // may either be regular
+                                          // or irregular. if it is
+                                          // regular and has a
+                                          // neighbor, then we
+                                          // visit the face twice,
+                                          // once from every
+                                          // side. let the one with
+                                          // the lower index do the
+                                          // work. if it is at the
+                                          // boundary, or if the
+                                          // face is irregular,
+                                          // then do the work below
+         if ((face->has_children() == false) &&
+             !cell->at_boundary(face_no) &&
+             (!cell->neighbor_is_coarser(face_no) &&
+              (cell->neighbor(face_no)->index() < cell->index() ||
+               (cell->neighbor(face_no)->index() == cell->index() &&
+                cell->neighbor(face_no)->level() < cell->level()))))
+           continue;
+         
+                                          // if the neighboring cell is less
+                                          // refined than the present one,
+                                          // then do nothing since we
+                                          // integrate over the subfaces when
+                                          // we visit the coarse cells.
+         if (cell->at_boundary(face_no) == false)
+           if (cell->neighbor_is_coarser(face_no))
+             continue;
+         
+                                          // if this face is part of the
+                                          // boundary but not of the neumann
+                                          // boundary -> nothing to
+                                          // do. However, to make things
+                                          // easier when summing up the
+                                          // contributions of the faces of
+                                          // cells, we enter this face into
+                                          // the list of faces with
+                                          // contribution zero.
+         const unsigned char boundary_indicator
+           = face->boundary_indicator();
+         if (face->at_boundary()
+             &&
+             (parallel_data.neumann_bc->find(boundary_indicator) ==
+              parallel_data.neumann_bc->end()))
+           {
+             local_face_integrals[face]
+               = std::vector<double> (n_solution_vectors, 0.);
+             continue;
+           }
+
+                                          // finally: note that we only have
+                                          // to do something if either the
+                                          // present cell is on the subdomain
+                                          // we care for (and the same for
+                                          // material_id), or if one of the
+                                          // neighbors behind the face is on
+                                          // the subdomain we care for
+         if ( ! ( ((subdomain_id == numbers::invalid_unsigned_int)
+                   ||
+                   (cell->subdomain_id() == subdomain_id))
+                  &&
+                  ((material_id == numbers::invalid_unsigned_int)
+                   ||
+                   (cell->material_id() == material_id))) )
+           {
+                                              // ok, cell is unwanted, but
+                                              // maybe its neighbor behind
+                                              // the face we presently work
+                                              // on? oh is there a face at
+                                              // all?
+             if (face->at_boundary())
+               continue;
+
+             bool care_for_cell = false;
+             if (face->has_children() == false)
+               care_for_cell |= ((cell->neighbor(face_no)->subdomain_id()
+                                  == subdomain_id) ||
+                                 (subdomain_id == numbers::invalid_unsigned_int))
+                                &&
+                                ((cell->neighbor(face_no)->material_id()
+                                  == material_id) ||
+                                 (material_id == numbers::invalid_unsigned_int));
+             else
+               {
+                 for (unsigned int sf=0; sf<face->n_children(); ++sf)
+                   if (((cell->neighbor_child_on_subface(face_no,sf)
+                         ->subdomain_id() == subdomain_id)
+                        &&
+                        (material_id ==
+                         numbers::invalid_unsigned_int))
+                       ||
+                       ((cell->neighbor_child_on_subface(face_no,sf)
+                         ->material_id() == material_id)
+                        &&
+                        (subdomain_id ==
+                         numbers::invalid_unsigned_int)))
+                     {
+                       care_for_cell = true;
+                       break;
+                     }
+               }
+
+                                              // so if none of the neighbors
+                                              // cares for this subdomain or
+                                              // material either, then try
+                                              // next face
+             if (care_for_cell == false)
+               continue;
+           }              
+
+                                          // so now we know that we care for
+                                          // this face, let's do something
+                                          // about it. first re-size the
+                                          // arrays we may use to the correct
+                                          // size:
+         parallel_data.resize (cell->active_fe_index());
+         
+         
+                                          // then do the actual integration
+         if (face->has_children() == false)
+                                            // if the face is a regular one,
+                                            // i.e.  either on the other side
+                                            // there is nirvana (face is at
+                                            // boundary), or the other side's
+                                            // refinement level is the same
+                                            // as that of this side, then
+                                            // handle the integration of
+                                            // these both cases together
+           integrate_over_regular_face (solutions,
+                                        parallel_data,
+                                        local_face_integrals,
+                                        cell, face_no,
+                                        parallel_data.fe_face_values_cell,
+                                        parallel_data.fe_face_values_neighbor);
+         
+         else
+                                            // otherwise we need to do some
+                                            // special computations which do
+                                            // not fit into the framework of
+                                            // the above function
+           integrate_over_irregular_face (solutions,
+                                          parallel_data,
+                                          local_face_integrals,
+                                          cell, face_no,
+                                          parallel_data.fe_face_values_cell,
+                                          parallel_data.fe_subface_values);
+       }
+    }    
+  }
 }
 
 
 
+
+
 #if deal_II_dimension == 1
 
 
@@ -469,90 +1399,12 @@ estimate (const Mapping<1,spacedim>                    &mapping,
 #else // #if deal_II_dimension !=1
 
 
-template <int dim, int spacedim>
-KellyErrorEstimator<dim, spacedim>::PerThreadData::
-PerThreadData (const unsigned int n_solution_vectors,
-              const unsigned int n_components,
-              const unsigned int max_n_q_points,
-               const unsigned int subdomain_id,
-               const unsigned int material_id)
-                :
-                subdomain_id (subdomain_id),
-                material_id (material_id)
-{
-                                  // Init the size of a lot of vectors needed
-                                  // in the calculations once per thread. we
-                                  // will later resize them as necessary, but
-                                  // for the moment just reserve the maximal
-                                  // memory necessary to avoid later
-                                  // re-allocation
-  normal_vectors.resize(max_n_q_points);
-  coefficient_values1.resize(max_n_q_points);
-  coefficient_values.resize(max_n_q_points);
-  JxW_values.resize(max_n_q_points);  
-
-  phi.resize(n_solution_vectors);
-  psi.resize(n_solution_vectors);
-  neighbor_psi.resize(n_solution_vectors);
-
-  for (unsigned int i=0; i<n_solution_vectors; ++i)
-    {
-      phi[i].resize(max_n_q_points);
-      psi[i].resize(max_n_q_points);
-      neighbor_psi[i].resize(max_n_q_points);
-
-      for (unsigned int qp=0;qp<max_n_q_points;++qp)
-       {
-         phi[i][qp].resize(n_components);
-         psi[i][qp].resize(n_components);
-         neighbor_psi[i][qp].resize(n_components);
-       }
-    }
-
-  for (unsigned int qp=0;qp<max_n_q_points;++qp)
-    coefficient_values[qp].reinit(n_components);
-}
-
 
 
+// the following function is still independent of dimension, but it
+// calls dimension dependent functions
 template <int dim, int spacedim>
-void
-KellyErrorEstimator<dim, spacedim>::PerThreadData::
-resize (const unsigned int n_components,
-       const unsigned int n_q_points)
-{
-  const unsigned int n_solution_vectors = phi.size();
-
-  normal_vectors.resize(n_q_points);
-  coefficient_values1.resize(n_q_points);
-  coefficient_values.resize(n_q_points);
-  JxW_values.resize(n_q_points);  
-
-  for (unsigned int i=0; i<n_solution_vectors; ++i)
-    {
-      phi[i].resize(n_q_points);
-      psi[i].resize(n_q_points);
-      neighbor_psi[i].resize(n_q_points);
-
-      for (unsigned int qp=0;qp<n_q_points;++qp)
-       {
-         phi[i][qp].resize(n_components);
-         psi[i][qp].resize(n_components);
-         neighbor_psi[i][qp].resize(n_components);
-       }
-    }
-
-  for (unsigned int qp=0;qp<n_q_points;++qp)
-    coefficient_values[qp].reinit(n_components);
-}
-
-
-
-
-// the following function is still independent of dimension, but it
-// calls dimension dependent functions
-template <int dim, int spacedim>
-template <typename InputVector, class DH>
+template <typename InputVector, class DH>
 void
 KellyErrorEstimator<dim, spacedim>::
 estimate (const Mapping<dim, spacedim>      &mapping,
@@ -644,274 +1496,6 @@ estimate (const DH                &dof_handler,
 
 
 
-template <int dim, int spacedim>
-template <typename InputVector, class DH>
-void KellyErrorEstimator<dim, spacedim>::
-estimate_some (const hp::MappingCollection<dim, spacedim>      &mapping,
-               const DH                                 &dof_handler,
-               const hp::QCollection<dim-1>             &quadrature,
-               const typename FunctionMap<dim>::type &neumann_bc,
-               const std::vector<const InputVector *> &solutions,
-              const std::pair<const std::vector<bool>*,const Function<dim>*> &component_mask_and_coefficients,
-               const std::pair<unsigned int, unsigned int> this_thread,
-               typename FaceIntegrals<DH>::type                       &face_integrals,
-               PerThreadData                       &per_thread_data)
-{
-  const std::vector<bool> &component_mask = *component_mask_and_coefficients.first;
-  const Function<dim>     *coefficients   = component_mask_and_coefficients.second;
-
-  const unsigned int n_solution_vectors = solutions.size();
-  const unsigned int n_components       = dof_handler.get_fe().n_components();
-  
-  const unsigned int subdomain_id = per_thread_data.subdomain_id;
-  const unsigned int material_id  = per_thread_data.material_id;
-
-  const hp::FECollection<dim> fe (dof_handler.get_fe());
-  
-                                  // make up a fe face values object
-                                  // for the restriction of the
-                                  // finite element function to a
-                                  // face, for the present cell and
-                                  // its neighbor. In principle we
-                                  // would only need one at a time,
-                                  // but this way we can have more
-                                  // fine grained access to what
-                                  // values really need to be
-                                  // computed (we need not compute
-                                  // all values on the neighbor
-                                  // cells, so using two objects
-                                  // gives us a performance gain).
-                                   //
-                                   // in debug mode, make sure that
-                                   // some data matches, so compute
-                                   // quadrature points always
-  hp::FEFaceValues<dim> fe_face_values_cell (mapping,
-                                            fe,
-                                            quadrature,
-                                            UpdateFlags(
-                                              update_gradients      |
-                                              update_JxW_values     |
-                                              ((!neumann_bc.empty() ||
-                                                (coefficients != 0))  ?
-                                               update_quadrature_points : 0) |
-                                              update_normal_vectors));
-  hp::FEFaceValues<dim> fe_face_values_neighbor (mapping,
-                                            fe,
-                                            quadrature,
-                                            update_gradients);
-  hp::FESubfaceValues<dim> fe_subface_values (mapping,
-                                             fe,
-                                             quadrature,
-                                             update_gradients);
-
-
-  typename DH::active_cell_iterator cell = dof_handler.begin_active();
-
-                                  // calculate the start cell for
-                                  // this thread. note that this way
-                                  // the threads work interleaved on
-                                  // successive cells, rather than on
-                                  // blocks of cells. the reason is
-                                  // that it takes vastly more time
-                                  // to work on cells with hanging
-                                  // nodes than on regular cells, but
-                                  // such cells are not evenly
-                                  // distributed across the range of
-                                  // cell iterators, so in order to
-                                  // have the different threads do
-                                  // approximately the same amount of
-                                  // work, we have to let them work
-                                  // interleaved to the effect of a
-                                  // pseudorandom distribution of the
-                                  // `hard' cells to the different
-                                  // threads.
-  for (unsigned int t=0; (t<this_thread.first) && (cell!=dof_handler.end());
-       ++t, ++cell)
-    ;
-  
-
-  
-                                  // loop over all cells for this thread
-  for (; cell!=dof_handler.end(); advance_by_n(cell,this_thread.second))
-    {
-      
-                                      // loop over all faces of this cell
-      for (unsigned int face_no=0;
-           face_no<GeometryInfo<dim>::faces_per_cell; ++face_no)
-       {
-         const typename DH::face_iterator
-           face=cell->face(face_no);
-
-                                          // make sure we do work
-                                          // only once: this face
-                                          // may either be regular
-                                          // or irregular. if it is
-                                          // regular and has a
-                                          // neighbor, then we
-                                          // visit the face twice,
-                                          // once from every
-                                          // side. let the one with
-                                          // the lower index do the
-                                          // work. if it is at the
-                                          // boundary, or if the
-                                          // face is irregular,
-                                          // then do the work below
-         if ((face->has_children() == false) &&
-             !cell->at_boundary(face_no) &&
-             (!cell->neighbor_is_coarser(face_no) &&
-              (cell->neighbor(face_no)->index() < cell->index() ||
-               (cell->neighbor(face_no)->index() == cell->index() &&
-                cell->neighbor(face_no)->level() < cell->level()))))
-           continue;
-         
-                                          // if we already visited
-                                          // this face: do
-                                          // nothing. only check
-                                          // component for first
-                                          // solution vector, as we
-                                          // treat them all at the
-                                          // same time
-         if (face_integrals[face][0] >=0)
-           continue;
-
-
-                                          // if the neighboring cell is less
-                                          // refined than the present one,
-                                          // then do nothing since we
-                                          // integrate over the subfaces when
-                                          // we visit the coarse cells.
-         if (cell->at_boundary(face_no) == false)
-           if (cell->neighbor_is_coarser(face_no))
-             continue;
-         
-                                          // if this face is part of the
-                                          // boundary but not of the neumann
-                                          // boundary -> nothing to
-                                          // do. However, to make things
-                                          // easier when summing up the
-                                          // contributions of the faces of
-                                          // cells, we enter this face into
-                                          // the list of faces with
-                                          // contribution zero.
-         const unsigned char boundary_indicator
-           = face->boundary_indicator();
-         if (face->at_boundary()
-             &&
-             neumann_bc.find(boundary_indicator)==neumann_bc.end()) 
-           {
-             for (unsigned int n=0; n<n_solution_vectors; ++n)
-               face_integrals[face][n] = 0;
-             continue;
-           }
-
-                                           // finally: note that we only have
-                                           // to do something if either the
-                                           // present cell is on the subdomain
-                                           // we care for (and the same for
-                                           // material_id), or if one of the
-                                           // neighbors behind the face is on
-                                           // the subdomain we care for
-          if ( ! ( ((subdomain_id == numbers::invalid_unsigned_int)
-                    ||
-                    (cell->subdomain_id() == subdomain_id))
-                   &&
-                   ((material_id == numbers::invalid_unsigned_int)
-                    ||
-                    (cell->material_id() == material_id))) )
-            {
-                                               // ok, cell is unwanted, but
-                                               // maybe its neighbor behind
-                                               // the face we presently work
-                                               // on? oh is there a face at
-                                               // all?
-              if (face->at_boundary())
-                continue;
-
-              bool care_for_cell = false;
-              if (face->has_children() == false)
-                care_for_cell |= ((cell->neighbor(face_no)->subdomain_id()
-                                   == subdomain_id) ||
-                                  (subdomain_id == numbers::invalid_unsigned_int))
-                                 &&
-                                 ((cell->neighbor(face_no)->material_id()
-                                   == material_id) ||
-                                  (material_id == numbers::invalid_unsigned_int));
-              else
-                {
-                  for (unsigned int sf=0; sf<face->n_children(); ++sf)
-                    if (((cell->neighbor_child_on_subface(face_no,sf)
-                          ->subdomain_id() == subdomain_id)
-                         &&
-                         (material_id ==
-                          numbers::invalid_unsigned_int))
-                        ||
-                        ((cell->neighbor_child_on_subface(face_no,sf)
-                          ->material_id() == material_id)
-                         &&
-                         (subdomain_id ==
-                          numbers::invalid_unsigned_int)))
-                      {
-                        care_for_cell = true;
-                        break;
-                      }
-                }
-
-                                               // so if none of the neighbors
-                                               // cares for this subdomain or
-                                               // material either, then try
-                                               // next face
-              if (care_for_cell == false)
-                continue;
-            }
-              
-
-                                           // so now we know that we care for
-                                           // this face, let's do something
-                                           // about it. first re-size the
-                                           // arrays we may use to the correct
-                                           // size:
-         per_thread_data.resize (n_components,
-                                 quadrature[cell->active_fe_index()]
-                                 .n_quadrature_points);
-         
-         
-                                          // then do the actual integration
-         if (face->has_children() == false)
-                                            // if the face is a regular one,
-                                            // i.e.  either on the other side
-                                            // there is nirvana (face is at
-                                            // boundary), or the other side's
-                                            // refinement level is the same
-                                            // as that of this side, then
-                                            // handle the integration of
-                                            // these both cases together
-           integrate_over_regular_face (dof_handler, quadrature,
-                                         neumann_bc, solutions, component_mask,
-                                         coefficients,
-                                         face_integrals,
-                                         per_thread_data,
-                                        cell, face_no,
-                                        fe_face_values_cell,
-                                        fe_face_values_neighbor);
-         
-         else
-                                            // otherwise we need to do some
-                                            // special computations which do
-                                            // not fit into the framework of
-                                            // the above function
-           integrate_over_irregular_face (dof_handler, quadrature,
-                                           solutions, component_mask,
-                                           coefficients,
-                                           face_integrals,
-                                           per_thread_data,
-                                          cell, face_no,
-                                          fe_face_values_cell,
-                                          fe_subface_values);
-       }
-    }
-}
-
-
 
 template <int dim, int spacedim>
 template <typename InputVector, class DH>
@@ -925,7 +1509,7 @@ estimate (const Mapping<dim, spacedim>                  &mapping,
           std::vector<Vector<float>*>              &errors,
           const std::vector<bool>                  &component_mask_,
           const Function<dim>                 *coefficients,
-          const unsigned int                   n_threads_,
+          const unsigned int                   ,
           const unsigned int                   subdomain_id,
           const unsigned int                   material_id)
 {
@@ -958,8 +1542,6 @@ estimate (const Mapping<dim, spacedim>                  &mapping,
     Assert (solutions[n]->size() == dof_handler.n_dofs(),
            ExcInvalidSolutionVector());
 
-  Assert (n_threads_ > 0, ExcInternalError());
-  
                                   // if no mask given: treat all components
   std::vector<bool> component_mask ((component_mask_.size() == 0)    ?
                                    std::vector<bool>(n_components, true) :
@@ -968,10 +1550,6 @@ estimate (const Mapping<dim, spacedim>                  &mapping,
   Assert (std::count(component_mask.begin(), component_mask.end(), true) > 0,
          ExcInvalidComponentMask());
          
-                                  // if NOT multithreaded, set
-                                  // n_threads to one
-  const unsigned int n_threads = (DEAL_II_USE_MT ? n_threads_ : 1);
-  
   const unsigned int n_solution_vectors = solutions.size();
 
                                   // Map of integrals indexed by
@@ -982,84 +1560,35 @@ estimate (const Mapping<dim, spacedim>                  &mapping,
                                   // loop over the cells and collect the
                                   // contributions of the different faces
                                   // of the cell.
-                                  // 
-                                  // The initial values for all faces
-                                  // are set to -10e20. It would cost
-                                  // a lot of time to synchronise the
-                                  // initialisation (i.e. the
-                                  // creation of new keys) of the map
-                                  // in multithreaded mode. Negative
-                                  // value indicates that the face
-                                  // has not yet been processed.
-  const double invalid_double = -10e20;
-  std::vector<double> default_face_integrals (n_solution_vectors,
-                                              invalid_double);
-  typename FaceIntegrals<DH>::type face_integrals;
-  for (typename DH::active_cell_iterator cell=dof_handler.begin_active();
-       cell!=dof_handler.end(); ++cell)
-    for (unsigned int face_no=0;
-         face_no<GeometryInfo<dim>::faces_per_cell;
-         ++face_no)
-      face_integrals[cell->face(face_no)] = default_face_integrals;
-
+  std::map<typename DH::face_iterator,std::vector<double> > face_integrals;
 
                                   // all the data needed in the error
                                   // estimator by each of the threads
                                   // is gathered in the following
                                   // stuctures
-                                  //
-                                  // note that if no component mask
-                                  // was given, then treat all
-                                  // components
-  std::vector<PerThreadData*> data_structures (n_threads);
-  for (unsigned int i=0; i<n_threads; ++i)
-    data_structures[i]
-      = new PerThreadData (solutions.size(),
-                           dof_handler.get_fe().n_components(),
-                           face_quadratures.max_n_quadrature_points(),
-                           subdomain_id, material_id);
-  
-                                  // split all cells into threads if
-                                  // multithreading is used and run
-                                  // the whole thing. use the
-                                  // function pointer variable to
-                                  // work around another nasty bug in
-                                  // icc7
-  Threads::ThreadGroup<> threads;
-  void (*estimate_some_ptr) (const hp::MappingCollection<dim,spacedim>                  &,
-                            const DH               &,
-                            const hp::QCollection<dim-1>             &,
-                            const typename FunctionMap<dim>::type &,
-                            const std::vector<const InputVector *> &,
-                            const std::pair<const std::vector<bool>*,const Function<dim>*> &,
-                            const std::pair<unsigned int, unsigned int>,
-                            typename FaceIntegrals<DH>::type                       &,
-                            PerThreadData                       &)
-    = &KellyErrorEstimator<dim, spacedim>::template estimate_some<InputVector,DH>;
-
-  hp::MappingCollection<dim,spacedim> mapping_collection;
-  mapping_collection.push_back (mapping);
-  
-  for (unsigned int i=0; i<n_threads; ++i)
-    threads += Threads::spawn (estimate_some_ptr)
-               (mapping_collection, dof_handler,
-                face_quadratures, neumann_bc, solutions,
-                std::make_pair(&component_mask, coefficients),
-                std::make_pair(i, n_threads),
-                face_integrals,
-                *data_structures[i]);
-  threads.join_all();
-
-                                  // delete the structures for the
-                                  // different threads again. the
-                                  // results are in the
-                                  // face_integrals object
-  for (unsigned int i=0; i<n_threads; ++i)
-    {
-      delete data_structures[i];
-      data_structures[i] = 0;
-    }
-  
+  const hp::MappingCollection<dim,spacedim> mapping_collection(mapping);
+  const internal::ParallelData<DH>
+    parallel_data (dof_handler.get_fe(),
+                  face_quadratures,
+                  mapping_collection,
+                  (!neumann_bc.empty() || (coefficients != 0)),
+                  solutions.size(),
+                  subdomain_id,
+                  material_id,
+                  &neumann_bc,
+                  component_mask,
+                  coefficients);
+  std::map<typename DH::face_iterator,std::vector<double> > sample_local_face_integrals;
+
+                                  // now let's work on all those cells:
+  WorkStream::run (dof_handler.begin_active(),
+                  static_cast<typename DH::active_cell_iterator>(dof_handler.end()),
+                  std_cxx1x::bind (&internal::estimate_one_cell<dim,spacedim,InputVector,DH>,
+                                   _1, _2, _3, std_cxx1x::ref(solutions)),
+                  std_cxx1x::bind (&internal::copy_local_to_global<DH>,
+                                   _1, std_cxx1x::ref(face_integrals)),
+                  parallel_data,
+                  sample_local_face_integrals);
   
                                   // finally add up the contributions of the
                                   // faces for each cell
@@ -1069,16 +1598,15 @@ estimate (const Mapping<dim, spacedim>                  &mapping,
   for (unsigned int n=0; n<n_solution_vectors; ++n)
     {
       (*errors[n]).reinit (dof_handler.get_tria().n_active_cells());
-      for (unsigned int i=0;i<dof_handler.get_tria().n_active_cells();++i)
+      for (unsigned int i=0; i<dof_handler.get_tria().n_active_cells(); ++i)
        (*errors[n])(i)=0;
     }
 
-  unsigned int present_cell=0;
-
                                    // now walk over all cells and collect
                                    // information from the faces. only do
                                    // something if this is a cell we care for
                                    // based on the subdomain id
+  unsigned int present_cell=0;
   for (typename DH::active_cell_iterator cell=dof_handler.begin_active();
        cell!=dof_handler.end();
        ++cell, ++present_cell)
@@ -1186,405 +1714,6 @@ void KellyErrorEstimator<dim, spacedim>::estimate (const DH
 
 
 
-template <int dim, int spacedim>
-template <typename InputVector, class DH>
-void KellyErrorEstimator<dim, spacedim>::
-integrate_over_regular_face (const DH                                 &dof_handler,
-                             const hp::QCollection<dim-1>             &quadrature,
-                             const typename FunctionMap<dim>::type &neumann_bc,
-                             const std::vector<const InputVector *> &solutions,
-                             const std::vector<bool>                  &component_mask,
-                             const Function<dim>                 *coefficients,
-                             typename FaceIntegrals<DH>::type                       &face_integrals,
-                             PerThreadData              &per_thread_data,
-                            const typename DH::active_cell_iterator &cell,
-                            const unsigned int          face_no,
-                            hp::FEFaceValues<dim>          &fe_face_values_cell,
-                            hp::FEFaceValues<dim>          &fe_face_values_neighbor)
-{
-  const typename DH::face_iterator face = cell->face(face_no);
-  const unsigned int n_q_points         = quadrature[cell->active_fe_index()].n_quadrature_points,
-                    n_components       = dof_handler.get_fe().n_components(),
-                    n_solution_vectors = solutions.size();
-  
-  
-                                  // initialize data of the restriction
-                                  // of this cell to the present face
-  fe_face_values_cell.reinit (cell, face_no,
-                             cell->active_fe_index());
-  
-                                  // get gradients of the finite element
-                                  // function on this cell
-  for (unsigned int n=0; n<n_solution_vectors; ++n)
-    fe_face_values_cell.get_present_fe_values()
-      .get_function_grads (*solutions[n], per_thread_data.psi[n]);
-  
-                                  // now compute over the other side of
-                                  // the face
-  if (face->at_boundary() == false)
-                                    // internal face; integrate jump
-                                    // of gradient across this face
-    {
-      Assert (cell->neighbor(face_no).state() == IteratorState::valid,
-             ExcInternalError());      
-      
-      const typename DH::active_cell_iterator neighbor = cell->neighbor(face_no);
-      
-                                      // find which number the
-                                      // current face has relative to
-                                      // the neighboring cell
-      const unsigned int neighbor_neighbor
-        = cell->neighbor_of_neighbor (face_no);
-      Assert (neighbor_neighbor<GeometryInfo<dim>::faces_per_cell,
-              ExcInternalError());
-      
-                                      // get restriction of finite element
-                                      // function of @p{neighbor} to the
-                                      // common face. in the hp case, use the
-                                      // quadrature formula that matches the
-                                      // one we would use for the present
-                                      // cell
-      fe_face_values_neighbor.reinit (neighbor, neighbor_neighbor,
-                                     cell->active_fe_index());
-      
-                                      // get gradients on neighbor cell
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       {
-         fe_face_values_neighbor.get_present_fe_values()
-           .get_function_grads (*solutions[n],
-                                per_thread_data.neighbor_psi[n]);
-      
-                                          // compute the jump in the gradients
-         for (unsigned int component=0; component<n_components; ++component)
-           for (unsigned int p=0; p<n_q_points; ++p)
-             per_thread_data.psi[n][p][component] -= per_thread_data.neighbor_psi[n][p][component];
-       }
-    }
-
-
-                                  // now psi contains the following:
-                                  // - for an internal face, psi=[grad u]
-                                  // - for a neumann boundary face,
-                                  //   psi=grad u
-                                  // each component being the
-                                  // mentioned value at one of the
-                                  // quadrature points
-  
-                                  // next we have to multiply this with
-                                  // the normal vector. Since we have
-                                  // taken the difference of gradients
-                                  // for internal faces, we may chose
-                                  // the normal vector of one cell,
-                                  // taking that of the neighbor
-                                  // would only change the sign. We take
-                                  // the outward normal.
-  
-  per_thread_data.normal_vectors =
-    fe_face_values_cell.get_present_fe_values().get_normal_vectors();
-  
-  for (unsigned int n=0; n<n_solution_vectors; ++n)
-    for (unsigned int component=0; component<n_components; ++component)
-      for (unsigned int point=0; point<n_q_points; ++point)
-       per_thread_data.phi[n][point][component]
-         = (per_thread_data.psi[n][point][component] *
-            per_thread_data.normal_vectors[point]);
-  
-                                  // if a coefficient was given: use that
-                                  // to scale the jump in the gradient
-  if (coefficients != 0)
-    {
-                                      // scalar coefficient
-      if (coefficients->n_components == 1)
-       {
-         
-         coefficients->value_list (fe_face_values_cell.get_present_fe_values()
-                                   .get_quadrature_points(),
-                                    per_thread_data.coefficient_values1);
-         for (unsigned int n=0; n<n_solution_vectors; ++n)
-           for (unsigned int component=0; component<n_components; ++component)
-             for (unsigned int point=0; point<n_q_points; ++point)
-               per_thread_data.phi[n][point][component] *=
-                 per_thread_data.coefficient_values1[point];
-       }
-      else
-                                          // vector-valued coefficient
-       {
-         coefficients->vector_value_list (fe_face_values_cell.get_present_fe_values()
-                                          .get_quadrature_points(),
-                                           per_thread_data.coefficient_values);
-         for (unsigned int n=0; n<n_solution_vectors; ++n)
-           for (unsigned int component=0; component<n_components; ++component)
-             for (unsigned int point=0; point<n_q_points; ++point)
-               per_thread_data.phi[n][point][component] *=
-                 per_thread_data.coefficient_values[point](component);
-       }
-    }
-
-
-  if (face->at_boundary() == true)
-                                    // neumann boundary face. compute
-                                    // difference between normal
-                                    // derivative and boundary function
-    {
-      const unsigned char boundary_indicator = face->boundary_indicator();
-      
-      Assert (neumann_bc.find(boundary_indicator) != neumann_bc.end(),
-             ExcInternalError ());
-                                      // get the values of the boundary
-                                      // function at the quadrature
-                                      // points
-      if (n_components == 1)
-       {
-         std::vector<double> g(n_q_points);
-         neumann_bc.find(boundary_indicator)->second
-           ->value_list (fe_face_values_cell.get_present_fe_values()
-                         .get_quadrature_points(), g);
-         
-         for (unsigned int n=0; n<n_solution_vectors; ++n)
-           for (unsigned int point=0; point<n_q_points; ++point)
-             per_thread_data.phi[n][point][0] -= g[point];
-       }
-      else
-       {
-         std::vector<Vector<double> > g(n_q_points, Vector<double>(n_components));
-         neumann_bc.find(boundary_indicator)->second
-           ->vector_value_list (fe_face_values_cell.get_present_fe_values()
-                                .get_quadrature_points(),
-                                g);
-         
-         for (unsigned int n=0; n<n_solution_vectors; ++n)
-           for (unsigned int component=0; component<n_components; ++component)
-             for (unsigned int point=0; point<n_q_points; ++point)
-               per_thread_data.phi[n][point][component] -= g[point](component);
-       }
-    }
-
-
-                                  // now phi contains the following:
-                                  // - for an internal face, phi=[a du/dn]
-                                  // - for a neumann boundary face,
-                                  //   phi=a du/dn-g
-                                  // each component being the
-                                  // mentioned value at one of the
-                                  // quadrature points
-
-  per_thread_data.JxW_values
-    = fe_face_values_cell.get_present_fe_values().get_JxW_values();
-  
-                                  // take the square of the phi[i]
-                                  // for integration, and sum up
-  std::vector<double> face_integral (n_solution_vectors, 0);
-  for (unsigned int n=0; n<n_solution_vectors; ++n)
-    for (unsigned int component=0; component<n_components; ++component)
-      if (component_mask[component] == true)
-       for (unsigned int p=0; p<n_q_points; ++p)
-         face_integral[n] += dealii::sqr(per_thread_data.phi[n][p][component]) *
-                             per_thread_data.JxW_values[p];
-
-                                  // double check that the element
-                                  // already exists and that it was
-                                  // not already written to
-  Assert (face_integrals.find (face) != face_integrals.end(),
-         ExcInternalError());
-  Assert (face_integrals[face][0] < 0, ExcInternalError());
-  
-  face_integrals[face] = face_integral;
-}
-
-
-
-template <int dim, int spacedim>
-template <typename InputVector, class DH>
-void KellyErrorEstimator<dim, spacedim>::
-integrate_over_irregular_face (const DH                                 &dof_handler,
-                               const hp::QCollection<dim-1>             &quadrature,
-                               const std::vector<const InputVector *> &solutions,
-                               const std::vector<bool>                  &component_mask,
-                               const Function<dim>                 *coefficients,
-                               typename FaceIntegrals<DH>::type                       &face_integrals,
-                               PerThreadData              &per_thread_data,
-                              const typename DH::active_cell_iterator &cell,
-                              const unsigned int          face_no,
-                              hp::FEFaceValues<dim>          &fe_face_values,
-                              hp::FESubfaceValues<dim>       &fe_subface_values)
-{
-  const typename DH::cell_iterator neighbor = cell->neighbor(face_no);
-  const unsigned int n_q_points         = quadrature[cell->active_fe_index()].n_quadrature_points,
-                    n_components       = dof_handler.get_fe().n_components(),
-                    n_solution_vectors = solutions.size();
-  const typename DH::face_iterator
-    face=cell->face(face_no);
-
-  Assert (neighbor.state() == IteratorState::valid, ExcInternalError());
-  Assert (face->has_children(), ExcInternalError());
-                                  // set up a vector of the gradients
-                                  // of the finite element function
-                                  // on this cell at the quadrature
-                                  // points
-                                  //
-                                  // let psi be a short name for
-                                  // [a grad u_h], where the second
-                                  // index be the component of the
-                                  // finite element, and the first
-                                  // index the number of the
-                                  // quadrature point
-  
-                                  // store which number @p{cell} has
-                                  // in the list of neighbors of
-                                  // @p{neighbor}
-  const unsigned int neighbor_neighbor
-    = cell->neighbor_of_neighbor (face_no);
-  Assert (neighbor_neighbor<GeometryInfo<dim>::faces_per_cell,
-          ExcInternalError());
-  
-                                  // loop over all subfaces
-  for (unsigned int subface_no=0; subface_no<face->n_children(); ++subface_no)
-    {
-                                      // get an iterator pointing to the
-                                      // cell behind the present subface
-      const typename DH::active_cell_iterator neighbor_child
-        = cell->neighbor_child_on_subface (face_no, subface_no);
-      Assert (!neighbor_child->has_children(),
-             ExcInternalError());
-      
-                                      // restrict the finite element
-                                      // on the present cell to the
-                                      // subface
-      fe_subface_values.reinit (cell, face_no, subface_no,
-                               cell->active_fe_index());
-
-                                      // restrict the finite element
-                                      // on the neighbor cell to the
-                                      // common @p{subface}.
-      fe_face_values.reinit (neighbor_child, neighbor_neighbor,
-                            cell->active_fe_index());
-
-                                       // store the gradient of the
-                                      // solution in psi
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       fe_subface_values.get_present_fe_values()
-         .get_function_grads (*solutions[n], per_thread_data.psi[n]);
-
-                                       // store the gradient from the
-                                      // neighbor's side in
-                                      // @p{neighbor_psi}
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       fe_face_values.get_present_fe_values()
-         .get_function_grads (*solutions[n], per_thread_data.neighbor_psi[n]);
-      
-                                      // compute the jump in the gradients
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       for (unsigned int component=0; component<n_components; ++component)
-         for (unsigned int p=0; p<n_q_points; ++p)
-           per_thread_data.psi[n][p][component] -=
-             per_thread_data.neighbor_psi[n][p][component];
-
-                                      // note that unlike for the
-                                      // case of regular faces
-                                      // (treated in the other
-                                      // function of this class), we
-                                      // have not to take care of
-                                      // boundary faces here, since
-                                      // they always are regular.
-      
-                                      // next we have to multiply this with
-                                      // the normal vector. Since we have
-                                      // taken the difference of gradients
-                                      // for internal faces, we may chose
-                                      // the normal vector of one cell,
-                                      // taking that of the neighbor
-                                      // would only change the sign. We take
-                                      // the outward normal.
-                                      //
-                                      // let phi be the name of the integrand
-     
-      per_thread_data.normal_vectors
-       = fe_face_values.get_present_fe_values().get_normal_vectors();
-
-
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       for (unsigned int component=0; component<n_components; ++component)
-         for (unsigned int point=0; point<n_q_points; ++point)
-           per_thread_data.phi[n][point][component] = (per_thread_data.psi[n][point][component]*
-                                            per_thread_data.normal_vectors[point]);
-      
-                                      // if a coefficient was given: use that
-                                      // to scale the jump in the gradient
-      if (coefficients != 0)
-       {
-                                          // scalar coefficient
-         if (coefficients->n_components == 1)
-           {
-             coefficients->value_list (fe_face_values.get_present_fe_values()
-                                       .get_quadrature_points(),
-                                        per_thread_data.coefficient_values1);
-             for (unsigned int n=0; n<n_solution_vectors; ++n)
-               for (unsigned int component=0; component<n_components; ++component)
-                 for (unsigned int point=0; point<n_q_points; ++point)
-                   per_thread_data.phi[n][point][component] *=
-                     per_thread_data.coefficient_values1[point];
-           }
-         else
-                                            // vector-valued coefficient
-           {
-             coefficients->vector_value_list (fe_face_values.get_present_fe_values()
-                                              .get_quadrature_points(),
-                                                   per_thread_data.coefficient_values);
-             for (unsigned int n=0; n<n_solution_vectors; ++n)
-               for (unsigned int component=0; component<n_components; ++component)
-                 for (unsigned int point=0; point<n_q_points; ++point)
-                   per_thread_data.phi[n][point][component] *=
-                     per_thread_data.coefficient_values[point](component);
-           }
-       }
-
-                                      // get the weights for the
-                                      // integration. note that it
-                                      // does not matter whether we
-                                      // take the JxW values from the
-                                      // fe_face_values or the
-                                      // fe_subface_values, as the
-                                      // first is on the small
-                                      // neighbor cell, while the
-                                      // latter is on the refined
-                                      // face of the big cell here
-      per_thread_data.JxW_values
-       = fe_face_values.get_present_fe_values().get_JxW_values();
-      
-                                      // take the square of the phi[i]
-                                      // for integration, and sum up
-      std::vector<double> face_integral (n_solution_vectors, 0);
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       for (unsigned int component=0; component<n_components; ++component)
-         if (component_mask[component] == true)
-           for (unsigned int p=0; p<n_q_points; ++p)
-             face_integral[n] += dealii::sqr(per_thread_data.phi[n][p][component]) *
-                                 per_thread_data.JxW_values[p];
-
-      face_integrals[neighbor_child->face(neighbor_neighbor)] = face_integral;
-    }
-
-
-                                  // finally loop over all subfaces to
-                                  // collect the contributions of the
-                                  // subfaces and store them with the
-                                  // mother face
-  std::vector<double> sum (n_solution_vectors, 0);
-  for (unsigned int subface_no=0; subface_no<face->n_children(); ++subface_no) 
-    {
-      Assert (face_integrals.find(face->child(subface_no)) !=
-             face_integrals.end(),
-             ExcInternalError());
-      Assert (face_integrals[face->child(subface_no)][0] >= 0,
-             ExcInternalError());
-      
-      for (unsigned int n=0; n<n_solution_vectors; ++n)
-       sum[n] += face_integrals[face->child(subface_no)][n];
-    }
-
-  face_integrals[face] = sum;
-}
-
 #endif
 
 
index 38368e01678087baba7752d75e0e3589fcd3488a..cc47a687d437563c0d4be81b78ee771f378745fc 100644 (file)
@@ -15,6 +15,7 @@
 #include <base/function.h>
 #include <base/quadrature.h>
 #include <base/thread_management.h>
+#include <base/work_stream.h>
 #include <base/multithread_info.h>
 #include <grid/tria_iterator.h>
 #include <grid/geometry_info.h>
 
 DEAL_II_NAMESPACE_OPEN
 
-template <typename DH>
-inline
-MatrixCreator::IteratorRange<DH>::
-IteratorRange (const active_cell_iterator &first,
-              const active_cell_iterator &second)
-               :
-               first (first),
-               second (second)
-{}
 
-
-
-template <typename DH>
-inline
-MatrixCreator::IteratorRange<DH>::IteratorRange (const iterator_pair &ip)
-               :
-               first (ip.first),
-               second (ip.second)
-{}
-
-
-
-
-template <int dim, typename number, int spacedim>
-void MatrixCreator::create_mass_matrix (const Mapping<dim, spacedim>       &mapping,
-                                       const DoFHandler<dim,spacedim>    &dof,
-                                       const Quadrature<dim>    &q,
-                                       SparseMatrix<number>     &matrix,
-                                       const Function<spacedim> * const coefficient)
+namespace internal
 {
-  Assert (matrix.m() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
-  Assert (matrix.n() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
-
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  const std::vector<std::pair<active_cell_iterator,active_cell_iterator> >
-    thread_ranges = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                                dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_mass_matrix_1_t) (const Mapping<dim, spacedim>       &mapping,
-                                         const DoFHandler<dim,spacedim>    &dof,
-                                         const Quadrature<dim>    &q,
-                                         SparseMatrix<number>     &matrix,
-                                         const Function<spacedim> * const coefficient,
-                                         const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                         Threads::ThreadMutex     &mutex);
-  create_mass_matrix_1_t p = &MatrixCreator::template create_mass_matrix_1<dim,number,spacedim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
+  namespace MatrixCreator
+  {
+    namespace AssemblerData
+    {
+      template <int dim,
+               int spacedim>
+      struct Scratch
+      {
+         Scratch (const FiniteElement<dim,spacedim> &fe,
+                  const UpdateFlags         update_flags,
+                  const Function<spacedim>      *coefficient,
+                  const Function<spacedim>      *rhs_function,
+                  const Quadrature<dim>    &quadrature,
+                  const Mapping<dim,spacedim>       &mapping)
+                         :
+                         fe_collection (fe),
+                         quadrature_collection (quadrature),
+                         mapping_collection (mapping),
+                         x_fe_values (mapping_collection,
+                                      fe_collection,
+                                      quadrature_collection,
+                                      update_flags),
+                         coefficient_values(quadrature_collection.max_n_quadrature_points()),
+                         coefficient_vector_values (quadrature_collection.max_n_quadrature_points(),
+                                                    dealii::Vector<double> (fe_collection.n_components())),
+                         rhs_values(quadrature_collection.max_n_quadrature_points()),
+                         rhs_vector_values (quadrature_collection.max_n_quadrature_points(),
+                                            dealii::Vector<double> (fe_collection.n_components())),
+                         coefficient (coefficient),
+                         rhs_function (rhs_function),
+                         update_flags (update_flags)
+           {}
+
+         Scratch (const ::dealii::hp::FECollection<dim,spacedim> &fe,
+                  const UpdateFlags         update_flags,
+                  const Function<spacedim>      *coefficient,
+                  const Function<spacedim>      *rhs_function,
+                  const ::dealii::hp::QCollection<dim>    &quadrature,
+                  const ::dealii::hp::MappingCollection<dim,spacedim>       &mapping)
+                         :
+                         fe_collection (fe),
+                         quadrature_collection (quadrature),
+                         mapping_collection (mapping),
+                         x_fe_values (mapping_collection,
+                                      fe_collection,
+                                      quadrature_collection,
+                                      update_flags),
+                         coefficient_values(quadrature_collection.max_n_quadrature_points()),
+                         coefficient_vector_values (quadrature_collection.max_n_quadrature_points(),
+                                                    dealii::Vector<double> (fe_collection.n_components())),
+                         rhs_values(quadrature_collection.max_n_quadrature_points()),
+                         rhs_vector_values(quadrature_collection.max_n_quadrature_points(),
+                                           dealii::Vector<double> (fe_collection.n_components())),
+                         coefficient (coefficient),
+                         rhs_function (rhs_function),
+                         update_flags (update_flags)
+           {}
+       
+         Scratch (const Scratch &data)
+                         :
+                         fe_collection (data.fe_collection),
+                         quadrature_collection (data.quadrature_collection),
+                         mapping_collection (data.mapping_collection),
+                         x_fe_values (mapping_collection,
+                                      fe_collection,
+                                      quadrature_collection,
+                                      data.update_flags),
+                         coefficient_values (data.coefficient_values),
+                         coefficient_vector_values (data.coefficient_vector_values),
+                         rhs_values (data.rhs_values),
+                         rhs_vector_values (data.rhs_vector_values),
+                         coefficient (data.coefficient),
+                         rhs_function (data.rhs_function),
+                         update_flags (data.update_flags)
+           {}
+       
+         const ::dealii::hp::FECollection<dim,spacedim> fe_collection;
+         const ::dealii::hp::QCollection<dim> quadrature_collection;
+         const ::dealii::hp::MappingCollection<dim,spacedim> mapping_collection;
+       
+         ::dealii::hp::FEValues<dim,spacedim> x_fe_values;
+       
+         std::vector<double>                  coefficient_values;
+         std::vector<dealii::Vector<double> > coefficient_vector_values;
+         std::vector<double>                  rhs_values;
+         std::vector<dealii::Vector<double> > rhs_vector_values;
+
+         const Function<spacedim>   *coefficient;
+         const Function<spacedim>   *rhs_function;
+
+         const UpdateFlags update_flags;
+      };
+
+
+      struct CopyData
+      {
+         std::vector<unsigned int> dof_indices;
+         FullMatrix<double>        cell_matrix;
+         dealii::Vector<double>    cell_rhs;
+      };
+      
 
-template <int dim, typename number, int spacedim>
-void MatrixCreator::create_mass_matrix_1 (const Mapping<dim, spacedim>       &mapping,
-                                         const DoFHandler<dim,spacedim>    &dof,
-                                         const Quadrature<dim>    &q,
-                                         SparseMatrix<number>     &matrix,
-                                         const Function<spacedim> * const coefficient,
-                                         const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                         Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_values | update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-  
-  FEValues<dim,spacedim> fe_values (mapping, dof.get_fe(), q, update_flags);
+      
+    }
     
-  const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                    n_q_points    = fe_values.n_quadrature_points;
-  const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-  const unsigned int n_components  = fe.n_components();
 
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-  
-  FullMatrix<number>  cell_matrix (dofs_per_cell, dofs_per_cell);
-  std::vector<double> coefficient_values (n_q_points);
-  std::vector<Vector<double> > coefficient_vector_values (n_q_points,
-                                                         Vector<double> (n_components));
-  
-  std::vector<unsigned int> dof_indices (dofs_per_cell);
-  
-  typename DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
+    template <int dim,
+             int spacedim,
+             typename CellIterator>
+    void mass_assembler (const CellIterator &cell,
+                        internal::MatrixCreator::AssemblerData::Scratch<dim,spacedim> &data,
+                        internal::MatrixCreator::AssemblerData::CopyData              &copy_data)
     {
-      fe_values.reinit (cell);
+      data.x_fe_values.reinit (cell);
+      const FEValues<dim,spacedim> &fe_values = data.x_fe_values.get_present_fe_values ();
       
-      cell_matrix = 0;
-      cell->get_dof_indices (dof_indices);
+      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
+                        n_q_points    = fe_values.n_quadrature_points;
+      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
+      const unsigned int n_components  = fe.n_components();
+
+      Assert(data.rhs_function == 0 ||
+            data.rhs_function->n_components==1 ||
+            data.rhs_function->n_components==n_components,
+            ::dealii::MatrixCreator::ExcComponentMismatch());
+      Assert(data.coefficient == 0 ||
+            data.coefficient->n_components==1 ||
+            data.coefficient->n_components==n_components,
+            ::dealii::MatrixCreator::ExcComponentMismatch());
+
+      copy_data.cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
+      copy_data.cell_matrix = 0;
+
+      copy_data.cell_rhs.reinit (dofs_per_cell);
+      copy_data.cell_rhs = 0;
       
-      if (coefficient != 0)
+      copy_data.dof_indices.resize (dofs_per_cell);
+      cell->get_dof_indices (copy_data.dof_indices);
+
+      if (data.rhs_function != 0)
        {
-         if (coefficient->n_components==1)
+         if (data.rhs_function->n_components==1)
+           {
+             data.rhs_values.resize (n_q_points);
+             data.rhs_function->value_list (fe_values.get_quadrature_points(),
+                                            data.rhs_values);
+           }
+         else
            {
-             // Version for variable coefficient with 1 component
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
+             data.rhs_vector_values.resize (n_q_points, 
+                                            dealii::Vector<double>(n_components));
+             data.rhs_function->vector_value_list (fe_values.get_quadrature_points(),
+                                                   data.rhs_vector_values);
+           }
+       }
+
+      if (data.coefficient != 0)
+       {
+         if (data.coefficient->n_components==1)
+           {
+             data.coefficient_values.resize (n_q_points);
+             data.coefficient->value_list (fe_values.get_quadrature_points(),
+                                           data.coefficient_values);
+           }
+         else
+           {
+             data.coefficient_vector_values.resize (n_q_points, 
+                                                    dealii::Vector<double>(n_components));
+             data.coefficient->vector_value_list (fe_values.get_quadrature_points(),
+                                                  data.coefficient_vector_values);
+           }
+       }
+
+
+      if (data.coefficient != 0)
+       {
+         if (data.coefficient->n_components==1)
+           {
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
                {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                 const unsigned int component_i =
+                   fe.system_to_component_index(i).first;
+                 for (unsigned int j=0; j<dofs_per_cell; ++j)
+                   if ((n_components==1) ||
+                       (fe.system_to_component_index(j).first ==
+                        component_i))
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       copy_data.cell_matrix(i,j)
+                         += (fe_values.shape_value(i,point) * 
+                             fe_values.shape_value(j,point) * 
+                             fe_values.JxW(point) *
+                             data.coefficient_values[point]);
+
+                 if (data.rhs_function != 0)
                    {
-                     const double v = fe_values.shape_value(i,point);
-                     const unsigned int component_i = 
-                       fe.system_to_component_index(i).first;
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       if ((n_components==1) ||
-                           (fe.system_to_component_index(j).first ==
-                            component_i))
-                         {
-                           const double u = fe_values.shape_value(j,point);
-                           cell_matrix(i,j) +=
-                             (u * v * weight * coefficient_values[point]);
-                         }
+                     if (data.rhs_function->n_components==1)
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_values[point] * fe_values.JxW(point);
+                     else
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_vector_values[point](component_i) * 
+                           fe_values.JxW(point);
                    }
                }
            }
          else
            {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
              if (fe.is_primitive ())
                {
-                 // Version for variable coefficient with multiple components
-                 for (unsigned int point=0; point<n_q_points; ++point)
+                 for (unsigned int i=0; i<dofs_per_cell; ++i)
                    {
-                     const double weight = fe_values.JxW(point);
-                     for (unsigned int i=0; i<dofs_per_cell; ++i)
+                     const unsigned int component_i =
+                       fe.system_to_component_index(i).first;
+                     for (unsigned int j=0; j<dofs_per_cell; ++j)
+                       if ((n_components==1) ||
+                           (fe.system_to_component_index(j).first ==
+                            component_i))
+                         for (unsigned int point=0; point<n_q_points; ++point)
+                           copy_data.cell_matrix(i,j) +=
+                             (fe_values.shape_value(i,point) * 
+                              fe_values.shape_value(j,point) * 
+                              fe_values.JxW(point) *
+                              data.coefficient_vector_values[point](component_i));
+
+                     if (data.rhs_function != 0)
                        {
-                         const double v = fe_values.shape_value(i,point);
-                         const unsigned int component_i=
-                           fe.system_to_component_index(i).first;
-                         for (unsigned int j=0; j<dofs_per_cell; ++j)
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(j).first == 
-                                component_i))
-                             {
-                               const double u = fe_values.shape_value(j,point);
-                               cell_matrix(i,j) +=
-                                 (u * v * weight *
-                                  coefficient_vector_values[point](component_i));
-                             }
+                         if (data.rhs_function->n_components==1)
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                               data.rhs_values[point] * fe_values.JxW(point);
+                         else
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                               data.rhs_vector_values[point](component_i) * 
+                               fe_values.JxW(point);
                        }
                    }
                }
              else
+                                                // non-primitive vector-valued FE
                {
-                 // Version for variable coefficient with multiple components and
-                 // vector values FE.
-                 for (unsigned int point=0; point<n_q_points; ++point)
-                   {
-                     const double weight = fe_values.JxW(point);
-                     for (unsigned int i=0; i<dofs_per_cell; ++i)
-                       for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
-                         if (fe.get_nonzero_components(i)[comp_i])
+                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                   for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
+                     if (fe.get_nonzero_components(i)[comp_i])
+                       {
+                         for (unsigned int j=0; j<dofs_per_cell; ++j)
+                           for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
+                             if (fe.get_nonzero_components(j)[comp_j])
+                               if (comp_i == comp_j)
+                                 for (unsigned int point=0; point<n_q_points; ++point)
+                                   copy_data.cell_matrix(i,j) += 
+                                     (fe_values.shape_value_component(i,point,comp_i) * 
+                                      fe_values.shape_value_component(j,point,comp_j) * 
+                                      fe_values.JxW(point) *
+                                      data.coefficient_vector_values[point](comp_i));
+                             
+                         if (data.rhs_function != 0)
                            {
-                             const double v = fe_values.shape_value_component(i,point,comp_i);
-                             for (unsigned int j=0; j<dofs_per_cell; ++j)
-                               for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
-                                 if (fe.get_nonzero_components(j)[comp_j])
-                                   {
-                                     const double u = fe_values.shape_value_component(j,point,comp_j);
-                                     if (comp_i == comp_j)
-                                       cell_matrix(i,j) +=
-                                         (u * v * weight *
-                                          coefficient_vector_values[point](comp_i));
-                                   }
+                             if (data.rhs_function->n_components==1)
+                               for (unsigned int point=0; point<n_q_points; ++point)
+                                 copy_data.cell_rhs(i) += 
+                                   fe_values.shape_value_component(i,point,comp_i) *
+                                   data.rhs_values[point] * fe_values.JxW(point);
+                             else
+                               for (unsigned int point=0; point<n_q_points; ++point)
+                                 copy_data.cell_rhs(i) += 
+                                   fe_values.shape_value_component(i,point,comp_i) *
+                                   data.rhs_vector_values[point](comp_i) * 
+                                   fe_values.JxW(point);
                            }
-                   }
+                       }
                }
            }
        }
       else
+                                        // no coefficient
        {
          if (fe.is_primitive ())
            {
-             // Version for primitive FEs
-             for (unsigned int point=0; point<n_q_points; ++point)
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
                {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                 const unsigned int component_i =
+                   fe.system_to_component_index(i).first;
+                 for (unsigned int j=0; j<dofs_per_cell; ++j)
+                   if ((n_components==1) ||
+                       (fe.system_to_component_index(j).first ==
+                        component_i))
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       copy_data.cell_matrix(i,j) +=
+                         (fe_values.shape_value(i,point) * 
+                          fe_values.shape_value(j,point) * 
+                          fe_values.JxW(point));
+
+                 if (data.rhs_function != 0)
                    {
-                     const double v = fe_values.shape_value(i,point); 
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       if ((n_components==1) ||
-                           (fe.system_to_component_index(i).first ==
-                            fe.system_to_component_index(j).first))
-                         {
-                           const double u = fe_values.shape_value(j,point);
-                           cell_matrix(i,j) += (u * v * weight);
-                         }
+                     if (data.rhs_function->n_components==1)
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_values[point] * fe_values.JxW(point);
+                     else
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_vector_values[point](component_i) * 
+                           fe_values.JxW(point);
                    }
                }
            }
          else
+                                            // non-primitive FE, no coefficient
            {
-             // Version for vector valued FEs
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
-                   for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
-                     if (fe.get_nonzero_components(i)[comp_i])
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
+               for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
+                 if (fe.get_nonzero_components(i)[comp_i])
+                   {
+                     for (unsigned int j=0; j<dofs_per_cell; ++j)
+                       for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
+                         if (fe.get_nonzero_components(j)[comp_j])
+                           if (comp_i == comp_j)
+                             for (unsigned int point=0; point<n_q_points; ++point)
+                               copy_data.cell_matrix(i,j) += 
+                                 (fe_values.shape_value_component(i,point,comp_i) * 
+                                  fe_values.shape_value_component(j,point,comp_j) * 
+                                  fe_values.JxW(point));
+
+                     if (data.rhs_function != 0)
                        {
-                         const double v = fe_values.shape_value_component(i,point,comp_i); 
-                         for (unsigned int j=0; j<dofs_per_cell; ++j)
-                           for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
-                             if (fe.get_nonzero_components(j)[comp_j])
-                               {
-                                 const double u = fe_values.shape_value_component(j,point,comp_j);
-                                 if (comp_i == comp_j)
-                                   cell_matrix(i,j) += (u * v * weight);
-                               }
+                         if (data.rhs_function->n_components==1)
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += 
+                               fe_values.shape_value_component(i,point,comp_i) *
+                               data.rhs_values[point] * fe_values.JxW(point);
+                         else
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += 
+                               fe_values.shape_value_component(i,point,comp_i) *
+                               data.rhs_vector_values[point](comp_i) * 
+                               fe_values.JxW(point);
                        }
-               }
+                   }
            }
        }
-                                      // transfer everything into the
-                                      // global object
-      mutex.acquire ();
-      matrix.add(dof_indices, cell_matrix);
-      mutex.release ();
-    };
-}
-
-
-template <int dim, typename number, int spacedim>
-void MatrixCreator::create_mass_matrix (const DoFHandler<dim,spacedim>    &dof,
-                                       const Quadrature<dim>    &q,
-                                       SparseMatrix<number>     &matrix,
-                                       const Function<spacedim> * const coefficient)
-{
-  Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_mass_matrix(StaticMappingQ1<dim,spacedim>::mapping, dof, q, matrix, coefficient);
-}
-
-
-template <int dim, typename number, int spacedim>
-void MatrixCreator::create_mass_matrix (const Mapping<dim, spacedim>       &mapping,
-                                       const DoFHandler<dim,spacedim>    &dof,
-                                       const Quadrature<dim>    &q,
-                                       SparseMatrix<number>     &matrix,
-                                       const Function<spacedim>      &rhs,
-                                       Vector<double>           &rhs_vector,
-                                       const Function<spacedim> * const coefficient)
-{
-  Assert (matrix.m() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
-  Assert (matrix.n() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
-
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_mass_matrix_2_t) (const Mapping<dim, spacedim>       &mapping,
-                                         const DoFHandler<dim,spacedim>    &dof,
-                                         const Quadrature<dim>    &q,
-                                         SparseMatrix<number>     &matrix,
-                                         const Function<spacedim>      &rhs,
-                                         Vector<double>           &rhs_vector,
-                                         const Function<spacedim> * const coefficient,
-                                         const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                         Threads::ThreadMutex     &mutex);
-  create_mass_matrix_2_t p = &MatrixCreator::template create_mass_matrix_2<dim,number,spacedim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, rhs,
-                                  rhs_vector, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, typename number, int spacedim>
-void
-MatrixCreator::create_mass_matrix_2 (const Mapping<dim, spacedim>       &mapping,
-                                    const DoFHandler<dim,spacedim>    &dof,
-                                    const Quadrature<dim>    &q,
-                                    SparseMatrix<number>     &matrix,
-                                    const Function<spacedim> &rhs,
-                                    Vector<double>           &rhs_vector,
-                                    const Function<spacedim> * const coefficient,
-                                    const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                    Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_values |
-                                        update_quadrature_points |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
+    }
 
-  FEValues<dim,spacedim> fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                    n_q_points    = fe_values.n_quadrature_points;
-  const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-  const unsigned int n_components  = fe.n_components();
 
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-  Assert (rhs.n_components == 1 ||
-         rhs.n_components == n_components,ExcComponentMismatch());
-  Assert (rhs_vector.size() == dof.n_dofs(),
-         ExcDimensionMismatch(rhs_vector.size(), dof.n_dofs()));
-  
-  FullMatrix<number>  cell_matrix (dofs_per_cell, dofs_per_cell);
-  Vector<double>      local_rhs (dofs_per_cell);
-  std::vector<double> coefficient_values (n_q_points);
-  std::vector<Vector<double> > coefficient_vector_values (n_q_points,
-                                                Vector<double> (n_components));
-  std::vector<double> rhs_values(n_q_points);
-  std::vector<Vector<double> > rhs_vector_values(n_q_points,
-                                                Vector<double> (n_components));
 
-  std::vector<unsigned int> dof_indices (dofs_per_cell);
+    template <int dim,
+             int spacedim,
+             typename CellIterator>
+    void laplace_assembler (const CellIterator &cell,
+                           internal::MatrixCreator::AssemblerData::Scratch<dim,spacedim> &data,
+                           internal::MatrixCreator::AssemblerData::CopyData              &copy_data)
+    {
+      data.x_fe_values.reinit (cell);
+      const FEValues<dim,spacedim> &fe_values = data.x_fe_values.get_present_fe_values ();      
 
-  typename DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
+      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
+                        n_q_points    = fe_values.n_quadrature_points;
+      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
+      const unsigned int n_components  = fe.n_components();
+
+      Assert(data.rhs_function == 0 ||
+            data.rhs_function->n_components==1 ||
+            data.rhs_function->n_components==n_components,
+            ::dealii::MatrixCreator::ExcComponentMismatch());
+      Assert(data.coefficient == 0 ||
+            data.coefficient->n_components==1 ||
+            data.coefficient->n_components==n_components,
+            ::dealii::MatrixCreator::ExcComponentMismatch());
+
+      copy_data.cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
+      copy_data.cell_matrix = 0;
+
+      copy_data.cell_rhs.reinit (dofs_per_cell);
+      copy_data.cell_rhs = 0;
+      
+      copy_data.dof_indices.resize (dofs_per_cell);
+      cell->get_dof_indices (copy_data.dof_indices);
 
-  for (; cell!=range.second; ++cell)
-    {
-      fe_values.reinit (cell);
 
-      cell_matrix = 0;
-      local_rhs = 0;
-      cell->get_dof_indices (dof_indices);
+      if (data.rhs_function != 0)
+       {
+         if (data.rhs_function->n_components==1)
+           {
+             data.rhs_values.resize (n_q_points);
+             data.rhs_function->value_list (fe_values.get_quadrature_points(),
+                                            data.rhs_values);
+           }
+         else
+           {
+             data.rhs_vector_values.resize (n_q_points, 
+                                            dealii::Vector<double>(n_components));
+             data.rhs_function->vector_value_list (fe_values.get_quadrature_points(),
+                                                   data.rhs_vector_values);
+           }
+       }
 
-                                  // value_list for one component rhs,
-                                  // vector_value_list otherwise
-      if (rhs.n_components==1)
-       rhs.value_list (fe_values.get_quadrature_points(), rhs_values);
-      else
-       rhs.vector_value_list (fe_values.get_quadrature_points(),
-                              rhs_vector_values);
+      if (data.coefficient != 0)
+       {
+         if (data.coefficient->n_components==1)
+           {
+             data.coefficient_values.resize (n_q_points);
+             data.coefficient->value_list (fe_values.get_quadrature_points(),
+                                           data.coefficient_values);
+           }
+         else
+           {
+             data.coefficient_vector_values.resize (n_q_points, 
+                                                    dealii::Vector<double>(n_components));
+             data.coefficient->vector_value_list (fe_values.get_quadrature_points(),
+                                                  data.coefficient_vector_values);
+           }
+       }
 
-                                  // Case with coefficient
-      if (coefficient != 0)
+      
+      if (data.coefficient != 0)
        {
-         if (coefficient->n_components == 1)
+         if (data.coefficient->n_components==1)
            {
-             // Version for variable coefficient with 1 component
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
                {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                 const unsigned int component_i =
+                   fe.system_to_component_index(i).first;
+                 for (unsigned int j=0; j<dofs_per_cell; ++j)
+                   if ((n_components==1) ||
+                       (fe.system_to_component_index(j).first ==
+                        component_i))
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       copy_data.cell_matrix(i,j)
+                         += (fe_values.shape_grad(i,point) * 
+                             fe_values.shape_grad(j,point) * 
+                             fe_values.JxW(point) *
+                             data.coefficient_values[point]);
+
+                 if (data.rhs_function != 0)
                    {
-                     const double v = fe_values.shape_value(i,point);
-                     const unsigned int component_i =
-                       fe.system_to_component_index(i).first;
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       if ((n_components==1) ||
-                           (fe.system_to_component_index(j).first ==
-                            component_i))
-                         {
-                           const double u = fe_values.shape_value(j,point);
-                           cell_matrix(i,j) +=
-                             (u * v * weight * coefficient_values[point]);
-                         }
-
-                     if (rhs.n_components == 1)
-                       local_rhs(i) += v * rhs_values[point] * weight;
+                     if (data.rhs_function->n_components==1)
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_values[point] * fe_values.JxW(point);
                      else
-                       local_rhs(i) += v * rhs_vector_values[point](component_i) 
-                                         * weight;
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_vector_values[point](component_i) * 
+                           fe_values.JxW(point);
                    }
                }
            }
          else
            {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
              if (fe.is_primitive ())
                {
-                 // Version for variable coefficient with multiple components
-                 for (unsigned int point=0; point<n_q_points; ++point)
+                 for (unsigned int i=0; i<dofs_per_cell; ++i)
                    {
-                     const double weight = fe_values.JxW(point);
-                     for (unsigned int i=0; i<dofs_per_cell; ++i)
+                     const unsigned int component_i =
+                       fe.system_to_component_index(i).first;
+                     for (unsigned int j=0; j<dofs_per_cell; ++j)
+                       if ((n_components==1) ||
+                           (fe.system_to_component_index(j).first ==
+                            component_i))
+                         for (unsigned int point=0; point<n_q_points; ++point)
+                           copy_data.cell_matrix(i,j) +=
+                             (fe_values.shape_grad(i,point) * 
+                              fe_values.shape_grad(j,point) * 
+                              fe_values.JxW(point) *
+                              data.coefficient_vector_values[point](component_i));
+
+                     if (data.rhs_function != 0)
                        {
-                         const double v = fe_values.shape_value(i,point);
-                         const unsigned int component_i=
-                           fe.system_to_component_index(i).first;
-                         for (unsigned int j=0; j<dofs_per_cell; ++j)
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(j).first == 
-                                component_i))
-                             {
-                               const double u = fe_values.shape_value(j,point);
-                               cell_matrix(i,j) +=
-                                 (u * v * weight *
-                                  coefficient_vector_values[point](component_i));
-                             }
-
-                         if (rhs.n_components == 1)
-                           local_rhs(i) += v * rhs_values[point] * weight;
+                         if (data.rhs_function->n_components==1)
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                               data.rhs_values[point] * fe_values.JxW(point);
                          else
-                           local_rhs(i) += v * rhs_vector_values[point](component_i) 
-                                             * weight;
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                               data.rhs_vector_values[point](component_i) * 
+                               fe_values.JxW(point);
                        }
                    }
                }
              else
+                                                // non-primitive vector-valued FE
                {
-                 // Version for variable coefficient with multiple components and
-                 // vector valued FE.
-                 for (unsigned int point=0; point<n_q_points; ++point)
-                   {
-                     const double weight = fe_values.JxW(point);
-                     for (unsigned int i=0; i<dofs_per_cell; ++i)
-                       for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
-                         if (fe.get_nonzero_components(i)[comp_i])
+                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                   for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
+                     if (fe.get_nonzero_components(i)[comp_i])
+                       {
+                         for (unsigned int j=0; j<dofs_per_cell; ++j)
+                           for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
+                             if (fe.get_nonzero_components(j)[comp_j])
+                               if (comp_i == comp_j)
+                                 for (unsigned int point=0; point<n_q_points; ++point)
+                                   copy_data.cell_matrix(i,j) += 
+                                     (fe_values.shape_grad_component(i,point,comp_i) * 
+                                      fe_values.shape_grad_component(j,point,comp_j) * 
+                                      fe_values.JxW(point) *
+                                      data.coefficient_vector_values[point](comp_i));
+                             
+                         if (data.rhs_function != 0)
                            {
-                             const double v = fe_values.shape_value_component(i,point,comp_i);
-                             for (unsigned int j=0; j<dofs_per_cell; ++j)
-                               for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
-                                 if (fe.get_nonzero_components(j)[comp_j])
-                                   {
-                                     const double u = fe_values.shape_value_component(j,point,comp_j);
-                                     if (comp_i == comp_j)
-                                       cell_matrix(i,j) +=
-                                         (u * v * weight *
-                                          coefficient_vector_values[point](comp_i));
-                                   }
-
-                             if (rhs.n_components == 1)
-                               local_rhs(i) += v * rhs_values[point] * weight;
+                             if (data.rhs_function->n_components==1)
+                               for (unsigned int point=0; point<n_q_points; ++point)
+                                 copy_data.cell_rhs(i) += 
+                                   fe_values.shape_value_component(i,point,comp_i) *
+                                   data.rhs_values[point] * fe_values.JxW(point);
                              else
-                               local_rhs(i) += v * rhs_vector_values[point](comp_i) 
-                                                 * weight;
+                               for (unsigned int point=0; point<n_q_points; ++point)
+                                 copy_data.cell_rhs(i) += 
+                                   fe_values.shape_value_component(i,point,comp_i) *
+                                   data.rhs_vector_values[point](comp_i) * 
+                                   fe_values.JxW(point);
                            }
-                   }
+                       }
                }
            }
        }
       else
+                                        // no coefficient
        {
          if (fe.is_primitive ())
            {
-             // Version for primitive FEs
-             for (unsigned int point=0; point<n_q_points; ++point)
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
                {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
+                 const unsigned int component_i =
+                   fe.system_to_component_index(i).first;
+                 for (unsigned int j=0; j<dofs_per_cell; ++j)
+                   if ((n_components==1) ||
+                       (fe.system_to_component_index(j).first ==
+                        component_i))
+                     for (unsigned int point=0; point<n_q_points; ++point)
+                       copy_data.cell_matrix(i,j) +=
+                         (fe_values.shape_grad(i,point) * 
+                          fe_values.shape_grad(j,point) * 
+                          fe_values.JxW(point));
+
+                 if (data.rhs_function != 0)
                    {
-                     const double v = fe_values.shape_value(i,point);
-                     const unsigned int component_i = 
-                       fe.system_to_component_index(i).first;
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       if ((n_components==1) ||
-                           (fe.system_to_component_index(j).first ==
-                            component_i))
-                         {
-                           const double u = fe_values.shape_value(j,point);
-                           cell_matrix(i,j) += (u * v * weight);
-                         }
-
-                     if (rhs.n_components == 1)
-                       local_rhs(i) += v * rhs_values[point] * weight;
+                     if (data.rhs_function->n_components==1)
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_values[point] * fe_values.JxW(point);
                      else
-                       local_rhs(i) += v * rhs_vector_values[point](component_i) 
-                                         * weight;
+                       for (unsigned int point=0; point<n_q_points; ++point)
+                         copy_data.cell_rhs(i) += fe_values.shape_value(i, point) *
+                           data.rhs_vector_values[point](component_i) * 
+                           fe_values.JxW(point);
                    }
                }
            }
          else
+                                            // non-primitive FE, no coefficient
            {
-             // Version for vector valued FEs
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
-                   for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
-                     if (fe.get_nonzero_components(i)[comp_i])
+             for (unsigned int i=0; i<dofs_per_cell; ++i)
+               for (unsigned int comp_i = 0; comp_i < n_components; ++comp_i)
+                 if (fe.get_nonzero_components(i)[comp_i])
+                   {
+                     for (unsigned int j=0; j<dofs_per_cell; ++j)
+                       for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
+                         if (fe.get_nonzero_components(j)[comp_j])
+                           if (comp_i == comp_j)
+                             for (unsigned int point=0; point<n_q_points; ++point)
+                               copy_data.cell_matrix(i,j) += 
+                                 (fe_values.shape_grad_component(i,point,comp_i) * 
+                                  fe_values.shape_grad_component(j,point,comp_j) * 
+                                  fe_values.JxW(point));
+
+                     if (data.rhs_function != 0)
                        {
-                         const double v = fe_values.shape_value_component(i,point,comp_i); 
-                         for (unsigned int j=0; j<dofs_per_cell; ++j)
-                           for (unsigned int comp_j = 0; comp_j < n_components; ++comp_j)
-                             if (fe.get_nonzero_components(j)[comp_j])
-                               {
-                                 const double u = fe_values.shape_value_component(j,point,comp_j);
-                                 if (comp_i == comp_j)
-                                   cell_matrix(i,j) += (u * v * weight);
-                               }
-
-                         if (rhs.n_components == 1)
-                           local_rhs(i) += v * rhs_values[point] * weight;
+                         if (data.rhs_function->n_components==1)
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += 
+                               fe_values.shape_value_component(i,point,comp_i) *
+                               data.rhs_values[point] * fe_values.JxW(point);
                          else
-                           local_rhs(i) += v * rhs_vector_values[point](comp_i) 
-                                             * weight;
+                           for (unsigned int point=0; point<n_q_points; ++point)
+                             copy_data.cell_rhs(i) += 
+                               fe_values.shape_value_component(i,point,comp_i) *
+                               data.rhs_vector_values[point](comp_i) * 
+                               fe_values.JxW(point);
                        }
-               }
+                   }
            }
        }
+    }
+    
 
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      matrix.add(dof_indices, cell_matrix);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       rhs_vector(dof_indices[i]) += local_rhs(i);
+    
+    template <typename MatrixType,
+             typename VectorType>
+    void copy_local_to_global (const AssemblerData::CopyData &data,
+                              MatrixType *matrix,
+                              VectorType *right_hand_side)
+    {
+      const unsigned int dofs_per_cell = data.dof_indices.size();
+
+      Assert (data.cell_matrix.m() == dofs_per_cell,
+             ExcInternalError());
+      Assert (data.cell_matrix.n() == dofs_per_cell,
+             ExcInternalError());
+      Assert ((right_hand_side == 0)
+             ||
+             (data.cell_rhs.size() == dofs_per_cell),
+             ExcInternalError());
+
+      matrix->add (data.dof_indices, data.cell_matrix);
+
+      if (right_hand_side != 0)
+       for (unsigned int i=0; i<dofs_per_cell; ++i)
+         (*right_hand_side)(data.dof_indices[i]) += data.cell_rhs(i);
     }
+  }
 }
 
 
+
+template <typename DH>
+inline
+MatrixCreator::IteratorRange<DH>::
+IteratorRange (const active_cell_iterator &first,
+              const active_cell_iterator &second)
+               :
+               first (first),
+               second (second)
+{}
+
+
+
+template <typename DH>
+inline
+MatrixCreator::IteratorRange<DH>::IteratorRange (const iterator_pair &ip)
+               :
+               first (ip.first),
+               second (ip.second)
+{}
+
+
+
+
+template <int dim, typename number, int spacedim>
+void MatrixCreator::create_mass_matrix (const Mapping<dim,spacedim>       &mapping,
+                                       const DoFHandler<dim,spacedim>    &dof,
+                                       const Quadrature<dim>    &q,
+                                       SparseMatrix<number>     &matrix,
+                                       const Function<spacedim> * const coefficient)
+{
+  Assert (matrix.m() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
+  Assert (matrix.n() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
+
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_values | update_JxW_values |
+                   (coefficient != 0 ? update_quadrature_points : UpdateFlags(0)),
+                   coefficient, /*rhs_function=*/0,
+                   q, mapping);
+  
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());
+
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::mass_assembler<dim, spacedim, typename DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<number>, Vector<double> >,
+                                   _1, &matrix, (Vector<double>*)0),
+                  assembler_data,
+                  copy_data);
+}
+
+
+
+template <int dim, typename number, int spacedim>
+void MatrixCreator::create_mass_matrix (const DoFHandler<dim,spacedim>    &dof,
+                                       const Quadrature<dim>    &q,
+                                       SparseMatrix<number>     &matrix,
+                                       const Function<spacedim> * const coefficient)
+{
+  Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
+  create_mass_matrix(StaticMappingQ1<dim,spacedim>::mapping, dof,
+                    q, matrix, coefficient);
+}
+
+
+
+template <int dim, typename number, int spacedim>
+void MatrixCreator::create_mass_matrix (const Mapping<dim,spacedim>       &mapping,
+                                       const DoFHandler<dim,spacedim>    &dof,
+                                       const Quadrature<dim>    &q,
+                                       SparseMatrix<number>     &matrix,
+                                       const Function<spacedim>      &rhs,
+                                       Vector<double>           &rhs_vector,
+                                       const Function<spacedim> * const coefficient)
+{
+  Assert (matrix.m() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
+  Assert (matrix.n() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
+
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_values |
+                   update_JxW_values | update_quadrature_points,
+                   coefficient, &rhs,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());  
+
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::mass_assembler<dim, spacedim, typename DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind(&internal::MatrixCreator::
+                                  copy_local_to_global<SparseMatrix<number>, Vector<double> >,
+                                  _1, &matrix, &rhs_vector),
+                  assembler_data,
+                  copy_data);
+}
+
+
+
 template <int dim, typename number, int spacedim>
 void MatrixCreator::create_mass_matrix (const DoFHandler<dim,spacedim>    &dof,
                                        const Quadrature<dim>    &q,
@@ -592,7 +790,7 @@ void MatrixCreator::create_mass_matrix (const DoFHandler<dim,spacedim>    &dof,
 
 
 template <int dim, typename number, int spacedim>
-void MatrixCreator::create_mass_matrix (const hp::MappingCollection<dim,spacedim>       &mapping,
+void MatrixCreator::create_mass_matrix (const hp::MappingCollection<dim,spacedim> &mapping,
                                        const hp::DoFHandler<dim,spacedim>    &dof,
                                        const hp::QCollection<dim>    &q,
                                        SparseMatrix<number>     &matrix,
@@ -603,164 +801,26 @@ void MatrixCreator::create_mass_matrix (const hp::MappingCollection<dim,spacedim
   Assert (matrix.n() == dof.n_dofs(),
          ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename hp::DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_mass_matrix_1_t) (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                         const hp::DoFHandler<dim,spacedim>    &dof,
-                                         const hp::QCollection<dim>    &q,
-                                         SparseMatrix<number>     &matrix,
-                                         const Function<spacedim> * const coefficient,
-                                         const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                         Threads::ThreadMutex     &mutex);
-  create_mass_matrix_1_t p = &MatrixCreator::template create_mass_matrix_1<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, typename number, int spacedim>
-void
-MatrixCreator::create_mass_matrix_1 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                    const hp::DoFHandler<dim,spacedim>    &dof,
-                                    const hp::QCollection<dim>    &q,
-                                    SparseMatrix<number>     &matrix,
-                                    const Function<spacedim> * const coefficient,
-                                    const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                    Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_values |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  hp::FEValues<dim,spacedim> x_fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int n_components  = dof.get_fe().n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-
-  FullMatrix<double>  cell_matrix;
-  std::vector<double> coefficient_values;
-  std::vector<Vector<double> > coefficient_vector_values;
-  
-  std::vector<unsigned int> dof_indices;
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_values | update_JxW_values |
+                   (coefficient != 0 ? update_quadrature_points : UpdateFlags(0)),
+                   coefficient, /*rhs_function=*/0,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());
   
-  typename hp::DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      x_fe_values.reinit (cell);
-      const FEValues<dim,spacedim> &fe_values = x_fe_values.get_present_fe_values ();
-
-      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                        n_q_points    = fe_values.n_quadrature_points;
-      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-
-      cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
-      coefficient_values.resize (n_q_points);
-      coefficient_vector_values.resize (n_q_points,
-                                       Vector<double> (n_components));
-      dof_indices.resize (dofs_per_cell);
-
-      
-      cell_matrix = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                   {
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(i).first ==
-                              fe.system_to_component_index(j).first))
-                           cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                                fe_values.shape_value(j,point) *
-                                                weight *
-                                                coefficient_values[point]);
-                       }
-                   }
-               }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                   {
-                     const unsigned int component_i=
-                       fe.system_to_component_index(i).first;              
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(j).first == component_i))
-                           cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                                fe_values.shape_value(j,point) *
-                                                weight *
-                                                coefficient_vector_values[point](component_i));
-                       }
-                   }
-               }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i) 
-             {
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                          fe_values.shape_value(j,point) *
-                                          weight);
-                 }
-             }
-         }
-
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-    }
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::mass_assembler<dim, spacedim, typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<number>, Vector<double> >,
+                                   _1, &matrix, (Vector<double>*)0),
+                  assembler_data,
+                  copy_data);
 }
 
 
@@ -772,7 +832,7 @@ void MatrixCreator::create_mass_matrix (const hp::DoFHandler<dim,spacedim>    &d
                                        const Function<spacedim> * const coefficient)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_mass_matrix(hp::StaticMappingQ1<dim>::mapping_collection, dof, q, matrix, coefficient);
+  create_mass_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q, matrix, coefficient);
 }
 
 
@@ -791,185 +851,26 @@ void MatrixCreator::create_mass_matrix (const hp::MappingCollection<dim,spacedim
   Assert (matrix.n() == dof.n_dofs(),
          ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename hp::DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_mass_matrix_2_t) (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                         const hp::DoFHandler<dim,spacedim>    &dof,
-                                         const hp::QCollection<dim>    &q,
-                                         SparseMatrix<number>     &matrix,
-                                         const Function<spacedim>      &rhs,
-                                         Vector<double>           &rhs_vector,
-                                         const Function<spacedim> * const coefficient,
-                                         const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                         Threads::ThreadMutex     &mutex);
-  create_mass_matrix_2_t p = &MatrixCreator::template create_mass_matrix_2<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, rhs,
-                                  rhs_vector, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, typename number, int spacedim>
-void
-MatrixCreator::create_mass_matrix_2 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                    const hp::DoFHandler<dim,spacedim>    &dof,
-                                    const hp::QCollection<dim>    &q,
-                                    SparseMatrix<number>     &matrix,
-                                    const Function<spacedim>      &rhs,
-                                    Vector<double>           &rhs_vector,
-                                    const Function<spacedim> * const coefficient,
-                                    const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                    Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_values    |
-                                        update_quadrature_points  |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  hp::FEValues<dim,spacedim> x_fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int n_components  = dof.get_fe().n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-
-  FullMatrix<double>  cell_matrix;
-  Vector<double>      local_rhs;
-  std::vector<double> rhs_values;
-  std::vector<double> coefficient_values;
-  std::vector<Vector<double> > coefficient_vector_values;
-  
-  std::vector<unsigned int> dof_indices;
-  
-  typename hp::DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      x_fe_values.reinit (cell);
-      const FEValues<dim,spacedim> &fe_values = x_fe_values.get_present_fe_values ();
-
-      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                        n_q_points    = fe_values.n_quadrature_points;
-      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-
-      cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
-      local_rhs.reinit (dofs_per_cell);
-      rhs_values.resize (fe_values.n_quadrature_points);
-      coefficient_values.resize (n_q_points);
-      coefficient_vector_values.resize (n_q_points,
-                                       Vector<double> (n_components));
-      dof_indices.resize (dofs_per_cell);
-
-      
-      cell_matrix = 0;
-      local_rhs = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      rhs.value_list (fe_values.get_quadrature_points(), rhs_values);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                   {
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(i).first ==
-                              fe.system_to_component_index(j).first))
-                           cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                                fe_values.shape_value(j,point) *
-                                                weight *
-                                                coefficient_values[point]);
-                       }
-                     local_rhs(i) += fe_values.shape_value(i,point) *
-                                     rhs_values[point] * weight;
-                   }
-               }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                   {
-                     const unsigned int component_i=
-                       fe.system_to_component_index(i).first;              
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(j).first == component_i))
-                           cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                                fe_values.shape_value(j,point) *
-                                                weight *
-                                                coefficient_vector_values[point](component_i));
-                       }                         
-                     local_rhs(i) += fe_values.shape_value(i,point) *
-                                     rhs_values[point] * weight;
-                   }
-               }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i) 
-             {
-               const double v = fe_values.shape_value(i,point);
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (fe_values.shape_value(i,point) *
-                                          fe_values.shape_value(j,point) *
-                                          weight);
-                 }
-               local_rhs(i) += v * rhs_values[point] * weight;
-             }
-         }
-
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       rhs_vector(dof_indices[i]) += local_rhs(i);
-    }
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_values |
+                   update_JxW_values | update_quadrature_points,
+                   coefficient, &rhs,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());
+
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::mass_assembler<dim, spacedim, typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<number>, Vector<double> >,
+                                   _1, &matrix, &rhs_vector),
+                  assembler_data,
+                  copy_data);
 }
 
 
@@ -983,7 +884,7 @@ void MatrixCreator::create_mass_matrix (const hp::DoFHandler<dim,spacedim>    &d
                                        const Function<spacedim> * const coefficient)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_mass_matrix(hp::StaticMappingQ1<dim>::mapping_collection, dof, q,
+  create_mass_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q,
                     matrix, rhs, rhs_vector, coefficient);
 }
 
@@ -1096,12 +997,15 @@ MatrixCreator::create_boundary_mass_matrix (const Mapping<dim, spacedim>  &mappi
        const IteratorRange<DoFHandler<dim,spacedim> >   range,
        Threads::ThreadMutex      &mutex);
   create_boundary_mass_matrix_1_t p = &MatrixCreator::template create_boundary_mass_matrix_1<dim,spacedim>;
+
+//TODO: Use WorkStream here  
   for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(Commons(mapping, dof, q), matrix,
-                                  boundary_functions, rhs_vector,
-                                  dof_to_boundary_mapping, coefficient,
-                                 component_mapping,
-                                  thread_ranges[thread], mutex);
+    threads += Threads::new_thread (p,
+                                   Commons(mapping, dof, q), matrix,
+                                   boundary_functions, rhs_vector,
+                                   dof_to_boundary_mapping, coefficient,
+                                   component_mapping,
+                                   thread_ranges[thread], mutex);
   threads.join_all ();  
 }
 
@@ -1168,7 +1072,7 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const Mapping<dim, spacedim> &,
                                          update_JxW_values |
                                          update_normal_vectors |
                                          update_quadrature_points);
-  FEFaceValues<dim> fe_values (mapping, fe, q, update_flags);
+  FEFaceValues<dim,spacedim> fe_values (mapping, fe, q, update_flags);
 
                                   // two variables for the coefficient,
                                   // one for the two cases indicated in
@@ -1449,7 +1353,7 @@ MatrixCreator::create_boundary_mass_matrix (const hp::MappingCollection<dim,spac
                                            const Function<spacedim> * const coefficient,
                                            std::vector<unsigned int> component_mapping)
 {
-  const hp::FECollection<dim> &fe_collection = dof.get_fe();
+  const hp::FECollection<dim,spacedim> &fe_collection = dof.get_fe();
   const unsigned int n_components  = fe_collection.n_components();
   
   Assert (matrix.n() == dof.n_boundary_dofs(boundary_functions),
@@ -1482,8 +1386,8 @@ MatrixCreator::create_boundary_mass_matrix (const hp::MappingCollection<dim,spac
     = Threads::split_range<active_cell_iterator> (dof.begin_active(),
                                                  dof.end(), n_threads);
 
-  typedef std_cxx1x::tuple<const hp::MappingCollection<dim>&,
-    const hp::DoFHandler<dim>&,
+  typedef std_cxx1x::tuple<const hp::MappingCollection<dim,spacedim>&,
+    const hp::DoFHandler<dim,spacedim>&,
     const hp::QCollection<dim-1>&> Commons;
   
                                   // mutex to synchronise access to
@@ -1502,12 +1406,15 @@ MatrixCreator::create_boundary_mass_matrix (const hp::MappingCollection<dim,spac
        const IteratorRange<hp::DoFHandler<dim,spacedim> >   range,
        Threads::ThreadMutex      &mutex);
   create_boundary_mass_matrix_1_t p = &MatrixCreator::template create_boundary_mass_matrix_1<dim,spacedim>;
+
+//TODO: Use WorkStream here  
   for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(Commons(mapping, dof, q), matrix,
-                                  boundary_functions, rhs_vector,
-                                  dof_to_boundary_mapping, coefficient,
-                                 component_mapping,
-                                  thread_ranges[thread], mutex);
+    threads += Threads::new_thread (p,
+                                   Commons(mapping, dof, q), matrix,
+                                   boundary_functions, rhs_vector,
+                                   dof_to_boundary_mapping, coefficient,
+                                   component_mapping,
+                                   thread_ranges[thread], mutex);
   threads.join_all ();  
 }
 
@@ -1528,8 +1435,8 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
                               const IteratorRange<hp::DoFHandler<dim,spacedim> >   range,
                               Threads::ThreadMutex      &mutex)
 {
-  const hp::MappingCollection<dim>& mapping = std_cxx1x::get<0>(commons);
-  const hp::DoFHandler<dim>& dof = std_cxx1x::get<1>(commons);
+  const hp::MappingCollection<dim,spacedim>& mapping = std_cxx1x::get<0>(commons);
+  const hp::DoFHandler<dim,spacedim>& dof = std_cxx1x::get<1>(commons);
   const hp::QCollection<dim-1>& q = std_cxx1x::get<2>(commons);
   const hp::FECollection<dim,spacedim> &fe_collection = dof.get_fe();
   const unsigned int n_components  = fe_collection.n_components();
@@ -1587,7 +1494,7 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
        {
          x_fe_values.reinit (cell, face);
 
-         const FEFaceValues<dim> &fe_values = x_fe_values.get_present_fe_values ();
+         const FEFaceValues<dim,spacedim> &fe_values = x_fe_values.get_present_fe_values ();
 
          const FiniteElement<dim,spacedim> &fe = cell->get_fe();
          const unsigned int dofs_per_cell = fe.dofs_per_cell;
@@ -1621,15 +1528,14 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
                            {
                              const double v = fe_values.shape_value(i,point);
                              for (unsigned int j=0; j<fe_values.dofs_per_cell; ++j)
-                               {
-                                 const double u = fe_values.shape_value(j,point);
-                                 if (fe.system_to_component_index(i).first ==
-                                     fe.system_to_component_index(j).first)
-                                   {
-                                     cell_matrix(i,j)
-                                       += (u * v * weight * coefficient_values[point]);
-                                   }
-                               }
+                               if (fe.system_to_component_index(i).first ==
+                                   fe.system_to_component_index(j).first)
+                                 {
+                                   const double u = fe_values.shape_value(j,point);
+                                   cell_matrix(i,j)
+                                     += (u * v * weight * coefficient_values[point]);
+                                 }
+                             
                              cell_vector(i) += v *
                                                rhs_values_system[point](
                                                  component_mapping[fe.system_to_component_index(i).first]) * weight;
@@ -1651,15 +1557,13 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
                              const unsigned int component_i=
                                fe.system_to_component_index(i).first;
                              for (unsigned int j=0; j<fe_values.dofs_per_cell; ++j)
-                               {
-                                 const double u = fe_values.shape_value(j,point);
-                                 if (fe.system_to_component_index(j).first ==
-                                     component_i)
-                                   {
-                                     cell_matrix(i,j) +=
-                                       (u * v * weight * coefficient_vector_values[point](component_i));
-                                   }
-                               } 
+                               if (fe.system_to_component_index(j).first ==
+                                   component_i)
+                                 {
+                                   const double u = fe_values.shape_value(j,point);
+                                   cell_matrix(i,j) +=
+                                     (u * v * weight * coefficient_vector_values[point](component_i));
+                                 }
                              cell_vector(i) += v * rhs_values_system[point](component_mapping[component_i]) * weight;
                            }
                        }
@@ -1673,14 +1577,12 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
                      {
                        const double v = fe_values.shape_value(i,point);
                        for (unsigned int j=0; j<fe_values.dofs_per_cell; ++j)
-                         {
-                           const double u = fe_values.shape_value(j,point);
-                           if (fe.system_to_component_index(i).first ==
-                               fe.system_to_component_index(j).first)
-                             {
-                               cell_matrix(i,j) += (u * v * weight);
-                             }
-                         }
+                         if (fe.system_to_component_index(i).first ==
+                             fe.system_to_component_index(j).first)
+                           {
+                             const double u = fe_values.shape_value(j,point);
+                             cell_matrix(i,j) += (u * v * weight);
+                           }
                        cell_vector(i) += v *
                                          rhs_values_system[point](
                                            fe.system_to_component_index(i).first) *
@@ -1875,207 +1777,29 @@ create_boundary_mass_matrix_1 (std_cxx1x::tuple<const hp::MappingCollection<dim,
              rhs_vector(dof_to_boundary_mapping[dofs[j]]) += cell_vector(j);
            else
              {
-                                                // compare here for relative
-                                                // smallness
-               Assert (std::fabs(cell_vector(j)) <= 1e-10 * max_diag_entry,
-                       ExcInternalError());
-             }
-       }
-}
-
-#endif
-
-template <int dim, int spacedim>
-void MatrixCreator::create_boundary_mass_matrix (const hp::DoFHandler<dim,spacedim>     &dof,
-                                                const hp::QCollection<dim-1>   &q,
-                                                SparseMatrix<double>      &matrix,
-                                                const typename FunctionMap<spacedim>::type &rhs,
-                                                Vector<double>            &rhs_vector,
-                                                std::vector<unsigned int> &dof_to_boundary_mapping,
-                                                const Function<spacedim> * const a,
-                                                std::vector<unsigned int> component_mapping)
-{
-  Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_boundary_mass_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q,
-                             matrix,rhs, rhs_vector, dof_to_boundary_mapping, a, component_mapping);
-}
-
-
-
-template <int dim, int spacedim>
-void MatrixCreator::create_laplace_matrix (const Mapping<dim, spacedim>       &mapping,
-                                          const DoFHandler<dim,spacedim>    &dof,
-                                          const Quadrature<dim>    &q,
-                                          SparseMatrix<double>     &matrix,
-                                          const Function<spacedim> * const coefficient)
-{
-  Assert (matrix.m() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
-  Assert (matrix.n() == dof.n_dofs(),
-         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
-
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_laplace_matrix_1_t) (const Mapping<dim, spacedim>       &mapping,
-                                            const DoFHandler<dim,spacedim>    &dof,
-                                            const Quadrature<dim>    &q,
-                                            SparseMatrix<double>     &matrix,
-                                            const Function<spacedim> * const coefficient,
-                                            const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                            Threads::ThreadMutex     &mutex);
-  create_laplace_matrix_1_t p = &MatrixCreator::template create_laplace_matrix_1<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, int spacedim>
-void MatrixCreator::create_laplace_matrix_1 (const Mapping<dim, spacedim>       &mapping,
-                                            const DoFHandler<dim,spacedim>    &dof,
-                                            const Quadrature<dim>    &q,
-                                            SparseMatrix<double>     &matrix,
-                                            const Function<spacedim> * const coefficient,
-                                            const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                            Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_JxW_values |
-                                        update_gradients);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  FEValues<dim,spacedim> fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                    n_q_points    = fe_values.n_quadrature_points;
-  const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-  const unsigned int n_components  = fe.n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-
-  FullMatrix<double>  cell_matrix (dofs_per_cell, dofs_per_cell);
-  std::vector<double> coefficient_values (n_q_points);
-  std::vector<Vector<double> > coefficient_vector_values (n_q_points,
-                                                         Vector<double> (n_components));
-  
-  std::vector<unsigned int> dof_indices (dofs_per_cell);
-  
-  typename DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      fe_values.reinit (cell);
-      
-      cell_matrix = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
-                   {
-                     const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(i).first ==
-                              fe.system_to_component_index(j).first))
-                           cell_matrix(i,j) += (Du * Dv * weight *
-                                                coefficient_values[point]);
-                       }
-                   }
-               }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-               {
-                 const double weight = fe_values.JxW(point);
-                 for (unsigned int i=0; i<dofs_per_cell; ++i)
-                   {
-                     const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                     const unsigned int component_i=
-                       fe.system_to_component_index(i).first;
-                     for (unsigned int j=0; j<dofs_per_cell; ++j)
-                       {
-                         const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                         if ((n_components==1) ||
-                             (fe.system_to_component_index(j).first == component_i))
-                           cell_matrix(i,j) += (Du * Dv * weight *
-                                                coefficient_vector_values[point](component_i));
-                         
-                       }
-                   }
-               }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i)
-             {
-               const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (Du * Dv * weight);
-                 }
+                                                // compare here for relative
+                                                // smallness
+               Assert (std::fabs(cell_vector(j)) <= 1e-10 * max_diag_entry,
+                       ExcInternalError());
              }
-         }
-    
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-    }
+       }
 }
 
-
+#endif
 
 template <int dim, int spacedim>
-void MatrixCreator::create_laplace_matrix (const DoFHandler<dim,spacedim>    &dof,
-                                          const Quadrature<dim>    &q,
-                                          SparseMatrix<double>     &matrix,
-                                          const Function<spacedim> * const coefficient)
+void MatrixCreator::create_boundary_mass_matrix (const hp::DoFHandler<dim,spacedim>     &dof,
+                                                const hp::QCollection<dim-1>   &q,
+                                                SparseMatrix<double>      &matrix,
+                                                const typename FunctionMap<spacedim>::type &rhs,
+                                                Vector<double>            &rhs_vector,
+                                                std::vector<unsigned int> &dof_to_boundary_mapping,
+                                                const Function<spacedim> * const a,
+                                                std::vector<unsigned int> component_mapping)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_laplace_matrix(StaticMappingQ1<dim>::mapping, dof, q, matrix, coefficient);
+  create_boundary_mass_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q,
+                             matrix,rhs, rhs_vector, dof_to_boundary_mapping, a, component_mapping);
 }
 
 
@@ -2085,8 +1809,6 @@ void MatrixCreator::create_laplace_matrix (const Mapping<dim, spacedim>       &m
                                           const DoFHandler<dim,spacedim>    &dof,
                                           const Quadrature<dim>    &q,
                                           SparseMatrix<double>     &matrix,
-                                          const Function<spacedim>      &rhs,
-                                          Vector<double>           &rhs_vector,
                                           const Function<spacedim> * const coefficient)
 {
   Assert (matrix.m() == dof.n_dofs(),
@@ -2094,176 +1816,75 @@ void MatrixCreator::create_laplace_matrix (const Mapping<dim, spacedim>       &m
   Assert (matrix.n() == dof.n_dofs(),
          ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_gradients  | update_JxW_values |
+                   (coefficient != 0 ? update_quadrature_points : UpdateFlags(0)),
+                   coefficient, /*rhs_function=*/0,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());  
+
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::laplace_assembler<dim, spacedim, typename DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<double>, Vector<double> >,
+                                   _1, &matrix, (Vector<double>*)0),
+                  assembler_data,
+                  copy_data);
+}
 
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
 
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_laplace_matrix_2_t) (const Mapping<dim, spacedim>       &mapping,
-                                            const DoFHandler<dim,spacedim>    &dof,
-                                            const Quadrature<dim>    &q,
-                                            SparseMatrix<double>     &matrix,
-                                            const Function<spacedim>      &rhs,
-                                            Vector<double>           &rhs_vector,
-                                            const Function<spacedim> * const coefficient,
-                                            const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                            Threads::ThreadMutex     &mutex);
-  create_laplace_matrix_2_t p = &MatrixCreator::template create_laplace_matrix_2<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, rhs,
-                                  rhs_vector, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
+
+template <int dim, int spacedim>
+void MatrixCreator::create_laplace_matrix (const DoFHandler<dim,spacedim>    &dof,
+                                          const Quadrature<dim>    &q,
+                                          SparseMatrix<double>     &matrix,
+                                          const Function<spacedim> * const coefficient)
+{
+  Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
+  create_laplace_matrix(StaticMappingQ1<dim,spacedim>::mapping, dof, q, matrix, coefficient);
 }
 
 
 
 template <int dim, int spacedim>
-void
-MatrixCreator::create_laplace_matrix_2 (const Mapping<dim, spacedim>       &mapping,
-                                       const DoFHandler<dim,spacedim>    &dof,
-                                       const Quadrature<dim>    &q,
-                                       SparseMatrix<double>     &matrix,
-                                       const Function<spacedim>      &rhs,
-                                       Vector<double>           &rhs_vector,
-                                       const Function<spacedim> * const coefficient,
-                                       const IteratorRange<DoFHandler<dim,spacedim> >  range,
-                                       Threads::ThreadMutex     &mutex)
+void MatrixCreator::create_laplace_matrix (const Mapping<dim, spacedim>       &mapping,
+                                          const DoFHandler<dim,spacedim>    &dof,
+                                          const Quadrature<dim>    &q,
+                                          SparseMatrix<double>     &matrix,
+                                          const Function<spacedim>      &rhs,
+                                          Vector<double>           &rhs_vector,
+                                          const Function<spacedim> * const coefficient)
 {
-  UpdateFlags update_flags = UpdateFlags(update_values    |
-                                        update_gradients |
-                                        update_quadrature_points  |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  FEValues<dim,spacedim> fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                    n_q_points    = fe_values.n_quadrature_points;
-  const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-  const unsigned int n_components  = fe.n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
+  Assert (matrix.m() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.m(), dof.n_dofs()));
+  Assert (matrix.n() == dof.n_dofs(),
+         ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  FullMatrix<double>  cell_matrix (dofs_per_cell, dofs_per_cell);
-  Vector<double>      local_rhs (dofs_per_cell);
-  std::vector<double> rhs_values (fe_values.n_quadrature_points);
-  std::vector<double> coefficient_values (n_q_points);
-  std::vector<Vector<double> > coefficient_vector_values (n_q_points,
-                                                         Vector<double> (n_components));
-  
-  std::vector<unsigned int> dof_indices (dofs_per_cell);
-  
-  typename DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      fe_values.reinit (cell);
-      
-      cell_matrix = 0;
-      local_rhs = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      rhs.value_list (fe_values.get_quadrature_points(), rhs_values);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const double v = fe_values.shape_value(i,point);
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(i).first ==
-                                fe.system_to_component_index(j).first))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_values[point]);
-                         }
-                       local_rhs(i) += v * rhs_values[point] * weight;  
-                     }
-                 }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const double v = fe_values.shape_value(i,point);
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       const unsigned int component_i=
-                         fe.system_to_component_index(i).first;                    
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(j).first == component_i))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_vector_values[point](component_i));
-                         }
-                       local_rhs(i) += v * rhs_values[point] * weight;
-                     }
-                 }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i) 
-             {
-               const double v = fe_values.shape_value(i,point);
-               const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (Du * Dv * weight);
-                 }
-               local_rhs(i) += v * rhs_values[point] * weight;
-             }
-         }
-
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       rhs_vector(dof_indices[i]) += local_rhs(i);
-    };
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_gradients  | update_values |
+                   update_JxW_values | update_quadrature_points,
+                   coefficient, &rhs,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::laplace_assembler<dim, spacedim, typename DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<double>, Vector<double> >,
+                                   _1, &matrix, &rhs_vector),
+                  assembler_data,
+                  copy_data);
 }
 
 
@@ -2277,7 +1898,7 @@ void MatrixCreator::create_laplace_matrix (const DoFHandler<dim,spacedim>    &do
                                           const Function<spacedim> * const coefficient)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_laplace_matrix(StaticMappingQ1<dim>::mapping, dof, q,
+  create_laplace_matrix(StaticMappingQ1<dim,spacedim>::mapping, dof, q,
                        matrix, rhs, rhs_vector, coefficient);
 }
 
@@ -2295,164 +1916,26 @@ void MatrixCreator::create_laplace_matrix (const hp::MappingCollection<dim,space
   Assert (matrix.n() == dof.n_dofs(),
          ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename hp::DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_laplace_matrix_1_t) (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                            const hp::DoFHandler<dim,spacedim>    &dof,
-                                            const hp::QCollection<dim>    &q,
-                                            SparseMatrix<double>     &matrix,
-                                            const Function<spacedim> * const coefficient,
-                                            const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                            Threads::ThreadMutex     &mutex);
-  create_laplace_matrix_1_t p = &MatrixCreator::template create_laplace_matrix_1<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, int spacedim>
-void
-MatrixCreator::create_laplace_matrix_1 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                       const hp::DoFHandler<dim,spacedim>    &dof,
-                                       const hp::QCollection<dim>    &q,
-                                       SparseMatrix<double>     &matrix,
-                                       const Function<spacedim> * const coefficient,
-                                       const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                       Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_gradients |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  hp::FEValues<dim,spacedim> x_fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int n_components  = dof.get_fe().n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-
-  FullMatrix<double>  cell_matrix;
-  std::vector<double> coefficient_values;
-  std::vector<Vector<double> > coefficient_vector_values;
-  
-  std::vector<unsigned int> dof_indices;
-  
-  typename hp::DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      x_fe_values.reinit (cell);
-      const FEValues<dim,spacedim> &fe_values = x_fe_values.get_present_fe_values ();
-
-      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                        n_q_points    = fe_values.n_quadrature_points;
-      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-
-      cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
-      coefficient_values.resize (n_q_points);
-      coefficient_vector_values.resize (n_q_points,
-                                       Vector<double> (n_components));
-      dof_indices.resize (dofs_per_cell);
-
-      
-      cell_matrix = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(i).first ==
-                                fe.system_to_component_index(j).first))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_values[point]);
-                         }
-                     }
-                 }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       const unsigned int component_i=
-                         fe.system_to_component_index(i).first;                    
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(j).first == component_i))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_vector_values[point](component_i));
-                         }
-                     }
-                 }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i) 
-             {
-               const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (Du * Dv * weight);
-                 }
-             }
-         }
-
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-    }
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_gradients  | update_JxW_values |
+                   (coefficient != 0 ? update_quadrature_points : UpdateFlags(0)),
+                   coefficient, /*rhs_function=*/0,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());  
+
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::laplace_assembler<dim, spacedim, typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<double>, Vector<double> >,
+                                   _1, &matrix, (Vector<double>*)0),
+                  assembler_data,
+                  copy_data);
 }
 
 
@@ -2464,7 +1947,7 @@ void MatrixCreator::create_laplace_matrix (const hp::DoFHandler<dim,spacedim>
                                           const Function<spacedim> * const coefficient)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_laplace_matrix(hp::StaticMappingQ1<dim>::mapping_collection, dof, q, matrix, coefficient);
+  create_laplace_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q, matrix, coefficient);
 }
 
 
@@ -2483,186 +1966,26 @@ void MatrixCreator::create_laplace_matrix (const hp::MappingCollection<dim,space
   Assert (matrix.n() == dof.n_dofs(),
          ExcDimensionMismatch (matrix.n(), dof.n_dofs()));
 
-  const unsigned int n_threads = multithread_info.n_default_threads;
-  Threads::ThreadGroup<> threads;
-
-                                  // define starting and end point
-                                  // for each thread
-  typedef typename hp::DoFHandler<dim,spacedim>::active_cell_iterator active_cell_iterator;  
-  std::vector<std::pair<active_cell_iterator,active_cell_iterator> > thread_ranges
-    = Threads::split_range<active_cell_iterator> (dof.begin_active(),
-                                                 dof.end(), n_threads);
-
-                                  // mutex to synchronise access to
-                                  // the matrix
-  Threads::ThreadMutex mutex;
-  
-                                  // then assemble in parallel
-  typedef void (*create_laplace_matrix_2_t) (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                            const hp::DoFHandler<dim,spacedim>    &dof,
-                                            const hp::QCollection<dim>    &q,
-                                            SparseMatrix<double>     &matrix,
-                                            const Function<spacedim>      &rhs,
-                                            Vector<double>           &rhs_vector,
-                                            const Function<spacedim> * const coefficient,
-                                            const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                            Threads::ThreadMutex     &mutex);
-  create_laplace_matrix_2_t p = &MatrixCreator::template create_laplace_matrix_2<dim>;
-  for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (p)(mapping, dof, q, matrix, rhs,
-                                  rhs_vector, coefficient,
-                                  thread_ranges[thread], mutex);
-  threads.join_all ();  
-}
-
-
-
-template <int dim, int spacedim>
-void
-MatrixCreator::create_laplace_matrix_2 (const hp::MappingCollection<dim,spacedim>       &mapping,
-                                       const hp::DoFHandler<dim,spacedim>    &dof,
-                                       const hp::QCollection<dim>    &q,
-                                       SparseMatrix<double>     &matrix,
-                                       const Function<spacedim>      &rhs,
-                                       Vector<double>           &rhs_vector,
-                                       const Function<spacedim> * const coefficient,
-                                       const IteratorRange<hp::DoFHandler<dim,spacedim> >  range,
-                                       Threads::ThreadMutex     &mutex)
-{
-  UpdateFlags update_flags = UpdateFlags(update_values    |
-                                        update_gradients |
-                                        update_quadrature_points  |
-                                        update_JxW_values);
-  if (coefficient != 0)
-    update_flags = UpdateFlags (update_flags | update_quadrature_points);
-
-  hp::FEValues<dim,spacedim> x_fe_values (mapping, dof.get_fe(), q, update_flags);
-    
-  const unsigned int n_components  = dof.get_fe().n_components();
-
-  Assert(coefficient == 0 ||
-        coefficient->n_components==1 ||
-        coefficient->n_components==n_components, ExcComponentMismatch());
-
-  FullMatrix<double>  cell_matrix;
-  Vector<double>      local_rhs;
-  std::vector<double> rhs_values;
-  std::vector<double> coefficient_values;
-  std::vector<Vector<double> > coefficient_vector_values;
-  
-  std::vector<unsigned int> dof_indices;
+  internal::MatrixCreator::AssemblerData::Scratch<dim, spacedim>
+    assembler_data (dof.get_fe(),
+                   update_gradients  | update_values |
+                   update_JxW_values | update_quadrature_points,
+                   coefficient, &rhs,
+                   q, mapping);
+  internal::MatrixCreator::AssemblerData::CopyData copy_data;
+  copy_data.cell_matrix.reinit (assembler_data.fe_collection.max_dofs_per_cell(),
+                        assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.cell_rhs.reinit (assembler_data.fe_collection.max_dofs_per_cell());
+  copy_data.dof_indices.resize (assembler_data.fe_collection.max_dofs_per_cell());
   
-  typename hp::DoFHandler<dim,spacedim>::active_cell_iterator cell = range.first;
-  for (; cell!=range.second; ++cell)
-    {
-      x_fe_values.reinit (cell);
-      const FEValues<dim,spacedim> &fe_values = x_fe_values.get_present_fe_values ();
-
-      const unsigned int dofs_per_cell = fe_values.dofs_per_cell,
-                        n_q_points    = fe_values.n_quadrature_points;
-      const FiniteElement<dim,spacedim>    &fe  = fe_values.get_fe();
-
-      cell_matrix.reinit (dofs_per_cell, dofs_per_cell);
-      local_rhs.reinit (dofs_per_cell);
-      rhs_values.resize (fe_values.n_quadrature_points);
-      coefficient_values.resize (n_q_points);
-      coefficient_vector_values.resize (n_q_points,
-                                       Vector<double> (n_components));
-      dof_indices.resize (dofs_per_cell);
-
-      
-      cell_matrix = 0;
-      local_rhs = 0;
-      cell->get_dof_indices (dof_indices);
-      
-      rhs.value_list (fe_values.get_quadrature_points(), rhs_values);
-      
-      if (coefficient != 0)
-       {
-         if (coefficient->n_components==1)
-           {
-             coefficient->value_list (fe_values.get_quadrature_points(),
-                                      coefficient_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const double v = fe_values.shape_value(i,point);
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(i).first ==
-                                fe.system_to_component_index(j).first))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_values[point]);
-                         }
-                       local_rhs(i) += v * rhs_values[point] * weight;
-                     }
-                 }
-           }
-         else
-           {
-             coefficient->vector_value_list (fe_values.get_quadrature_points(),
-                                             coefficient_vector_values);
-             for (unsigned int point=0; point<n_q_points; ++point)
-                 {
-                   const double weight = fe_values.JxW(point);
-                   for (unsigned int i=0; i<dofs_per_cell; ++i) 
-                     {
-                       const double v = fe_values.shape_value(i,point);
-                       const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-                       const unsigned int component_i=
-                         fe.system_to_component_index(i).first;                    
-                       for (unsigned int j=0; j<dofs_per_cell; ++j)
-                         {
-                           const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                           if ((n_components==1) ||
-                               (fe.system_to_component_index(j).first == component_i))
-                             cell_matrix(i,j) += (Du * Dv * weight *
-                                                  coefficient_vector_values[point](component_i));
-                         }
-                       local_rhs(i) += v * rhs_values[point] * weight;
-                     }
-                 }
-           }
-       }
-      else
-       for (unsigned int point=0; point<n_q_points; ++point)
-         {
-           const double weight = fe_values.JxW(point);
-           for (unsigned int i=0; i<dofs_per_cell; ++i) 
-             {
-               const double v = fe_values.shape_value(i,point);
-               const Tensor<1,dim>& Dv = fe_values.shape_grad(i,point);
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 {
-                   const Tensor<1,dim>& Du = fe_values.shape_grad(j,point);
-                   if ((n_components==1) ||
-                       (fe.system_to_component_index(i).first ==
-                        fe.system_to_component_index(j).first))
-                     cell_matrix(i,j) += (Du * Dv * weight);
-                 }
-               local_rhs(i) += v * rhs_values[point] * weight;
-             }
-         }
-
-                                      // transfer everything into the
-                                      // global object. lock the
-                                      // matrix meanwhile
-      Threads::ThreadMutex::ScopedLock lock (mutex);
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       for (unsigned int j=0; j<dofs_per_cell; ++j)
-         if ((n_components==1) ||
-             (fe.system_to_component_index(i).first ==
-              fe.system_to_component_index(j).first))
-           matrix.add (dof_indices[i], dof_indices[j],
-                       cell_matrix(i,j));
-      for (unsigned int i=0; i<dofs_per_cell; ++i)
-       rhs_vector(dof_indices[i]) += local_rhs(i);
-    }
+  WorkStream::run (dof.begin_active(),
+                  static_cast<typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>(dof.end()),
+                  &internal::MatrixCreator::laplace_assembler<dim, spacedim, typename hp::DoFHandler<dim,spacedim>::active_cell_iterator>,
+                  std_cxx1x::bind (&internal::MatrixCreator::
+                                   copy_local_to_global<SparseMatrix<double>, Vector<double> >,
+                                   _1, &matrix, &rhs_vector),
+                  assembler_data,
+                  copy_data);
 }
 
 
@@ -2676,7 +1999,7 @@ void MatrixCreator::create_laplace_matrix (const hp::DoFHandler<dim,spacedim>
                                           const Function<spacedim> * const coefficient)
 {
   Assert (DEAL_II_COMPAT_MAPPING, ExcCompatibility("mapping"));
-  create_laplace_matrix(hp::StaticMappingQ1<dim>::mapping_collection, dof, q,
+  create_laplace_matrix(hp::StaticMappingQ1<dim,spacedim>::mapping_collection, dof, q,
                        matrix, rhs, rhs_vector, coefficient);
 }
 
index 64404ff0cc15b38815617c0f6fb894d5f8aea557..534c589adcf2bb36b12a3c0538ae14c39acfde6a 100644 (file)
@@ -212,10 +212,10 @@ void TimeDependent::end_sweep (const unsigned int n_threads)
       void (TimeDependent::*p) (const unsigned int, const unsigned int)
         = &TimeDependent::end_sweep;
       for (unsigned int i=0; i<n_threads; ++i)
-        threads += Threads::spawn (*this, p)(i*stride,
-                                             (i == n_threads-1 ?
-                                              timesteps.size() :
-                                              (i+1)*stride));
+        threads += Threads::new_thread (p, *this, i*stride,
+                                       (i == n_threads-1 ?
+                                        timesteps.size() :
+                                        (i+1)*stride));
       threads.join_all();
     }
   else
index b659f70bb2e0fb9a0f670f14346ab7b091c6aece..9ce2b295f7d71a17f751327f9618fb60bdbf06fb 100644 (file)
@@ -71,7 +71,7 @@ makefiles.html: Makefile makefiles.1.html $D/common/Make.global_options makefile
        @echo '<li><code>F77LIBS=$(F77LIBS)</code>' >> $@
        @echo '<li><code>ACE_ROOT=$(ACE_ROOT)</code>' >> $@
        @echo '<li><code>lib-ACE=$(lib-ACE)</code>' >> $@
-       @echo '<li><code>with-multithreading=$(with-multithreading)</code>' >> $@
+       @echo '<li><code>enable-threads=$(enable-threads)</code>' >> $@
        @echo '<li><code>TECIO_INCLUDE=$(TECIO_INCLUDE)</code>' >> $@
        @echo '<li><code>TECIO_LIBRARY=$(TECIO_LIBRARY)</code>' >> $@
        @cat makefiles.2.html >> $@
index 316d049f4bf8bd404567df75dc25ac75aeb7a9a0..1c813c474e8ca8fe8025f56019549f97c43963b6 100644 (file)
@@ -64,17 +64,6 @@ libs.o   = $(lib-deal2-$(deal_II_dimension)d.o) \
 
 
 
-# Define a nifty string to indicate in the output of the compile
-# commands whether the program is compiled in multithread mode or not:
-ifneq ($(with-multithreading),no)
-  MT = MT
-else
-  MT = ==
-endif
-
-
-
-
 # Now use the information from above to define the set of libraries to
 # link with and the flags to be passed to the compiler:
 ifeq ($(debug-mode),on)
index 68668cab366a48743b81021903f480445570af66..ece06b10e5056166116e0d934ec532836e9b13bb 100644 (file)
     <li><h4>Variables in multithreading mode</h4>
  
     <dl>
-      <dt> <code>with-multithreading</code> </dt>
+      <dt> <code>enable-threads</code> </dt>
       <dd> <p>
            This symbol is 'no' if the flag was not given
            to ./configure and 'posix' otherwise
index 5e2644c4e4c744c08990f4feda6d60284739f8de..5049c14f5bdd4f25933c64905e6f3856c4269ef3 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2006 by the deal.II authors
+//    Copyright (C) 2006, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 //-------------------------------------------------------------------------
 
 /**
- * @defgroup threads Multithreading
+ * @defgroup threads Parallel computing with multiple processors
+ *
+ * @brief A module discussing the use of parallelism on shared memory
+ * machines. See the detailed documentation and
+ * @ref MTToC "Table of Contents" below the lenghty list of members
+ * of this module.
  *
  * On machines with more than one processor (or multicore processors),
  * it is often profitable to run several parts of the computations in
- * parallel. For example, one could have several threads running in
- * parallel, each of which assembles the cell matrices of a subset of
+ * %parallel. For example, one could have several threads running in
+ * %parallel, each of which assembles the cell matrices of a subset of
  * the triangulation and then writes them into the global
  * matrix. Since assembling matrices is often an expensive operation,
  * this frequently leads to significant savings in compute time on
  * multiprocessor machines.
  *
- * In a similar way, it is often also profitable to use multiple
- * threads on a single-CPU system if a significant amount of input or
- * output tasks has to be performed. In such cases, the program
- * usually has to wait for disks or network storages to provide the
- * requested data, or to flush buffers. If this is done on a separate
- * thread, other threads of the program can continue to do other, more
- * interesting things at the same time, using the CPU downtime.
- *
- * Support for this model of computations, i.e. using multiple threads
- * on a shared-memory machine (SMP machine) is provided mainly through
- * the Threads namespace that offers functions to create new threads
- * as well as synchronisation primitives. The MultithreadInfo class
- * allows to query certain properties of the system, such as the
- * number of CPUs.  The use of these classes is explained in the
- * step-9, step-13 and step-14 tutorial programs.
+ * deal.II supports operations running in %parallel on on shared-memory (SMP)
+ * machines through the functions and classes in the Threads namespace. The
+ * MultithreadInfo class allows to query certain properties of the system,
+ * such as the number of CPUs. These facilities for %parallel computing are
+ * described in the following. The step-9, step-13 and step-14 tutorial
+ * programs also show their use in practice.
  *
  * On the other hand, programs running on distributed memory machines
- * (i.e. clusters) need a different programming model built on top of
- * MPI and PETSc that is described in the step-17 and later example
- * programs.
+ * (i.e. clusters) need a different programming model built on top of MPI and
+ * PETSc or Trilinos. This is described in the step-17, step-18 and step-32
+ * example programs.
+ *
+ * @anchor MTToC
+ * <table class="tutorial" width="50%">
+ * <tr><th><b>%Table of contents</b></th></tr>
+ * <tr><td width="100%" valign="top">
+ * <ol>
+ *  <li> @ref MTTasks "Task-based parallelism"
+ *  <li> @ref MTUsing "Using tasks from within deal.II"
+ *  <li> @ref MTHow "How scheduling tasks works and when task-based programming is not efficient"
+ *  <li> @ref MTSimpleLoops "Abstractions for tasks: Simple loops"
+ *  <li> @ref MTComplexLoops "Abstractions for tasks: More complex loops"
+ *  <li> @ref MTWorkStream "Abstractions for tasks: Work streams"
+ *  <li> @ref MTThreads "Thread-based parallelism"
+ * </ol> </td> </tr> </table>
+ *
+ *
+ * @anchor MTTasks
+ * <h3>Task-based parallelism</h3>
+ *
+ * The traditional view of parallelism on shared memory machines has been to
+ * decompose a program into <i>threads</i>, i.e. running different parts of
+ * the program in %parallel <i>at the same time</i> (if there are more threads
+ * than processor cores on your machine, the operating system will run each
+ * thread round-robin for a brief amount of time before switching execution to
+ * another thread, thereby simulating that threads run
+ * concurrently). deal.II's facilities for threads are described below (see
+ * @ref MTThreads "Thread-based parallelism"), but we would first like to
+ * discuss an abstraction that is often more suitable than threads:
+ * <i>tasks</i>.
+ *
+ * Tasks are essentially the individual parts of a program. Some of them are
+ * independent, whereas others depend on previous tasks to be completed
+ * first. By way of example, consider the typical layout of a part of the
+ * <code>setup_dofs</code> function that most of the tutorial programs have:
+ * @code
+1  dof_handler.distribute_dofs (fe);  
+2  DoFTools::make_hanging_node_constraints (dof_handler, hanging_node_constraints);
+3  DoFTools::make_sparsity_pattern (dof_handler, sparsity_pattern);
+4  hanging_node_constraints.condense (sparsity_pattern);
+ * @endcode
+ *
+ * Here, each of the operations require a significant amount of
+ * computations. But note that not all of them depend on each other: clearly
+ * we can not run statements 2-4 before 1, and 4 needs to wait for the
+ * completion of statements 2 and 3. But statements 2 and 3 are independent:
+ * they could be run in any order, or in %parallel. In essence, we have
+ * identified four <i>tasks</i>, some of which are dependent on each other,
+ * whereas others are independent. In the current example, tasks are
+ * identified with individual C++ statements, but oftentimes they more
+ * generally coincide with entire code blocks.
+ *
+ * The point here is this: To exploit the independence of tasks 2 and 3, we
+ * could start two threads and run each task on its own thread; we would then
+ * wait for the two threads to finish (an operation called "joining a thread")
+ * and go on with statement 4. As discussed in more detail below, code to
+ * achieve this would look like this:
+ * @code
+   dof_handler.distribute_dofs (fe);
+
+   Threads::Thread<void>
+     thread_1 = Threads::new_thread (&DoFTools::make_hanging_node_constraints,
+                                     dof_handler, hanging_node_constraints);
+   Threads::Thread<void>
+     thread_2 = Threads::new_thread (&DoFTools::make_sparsity_pattern,
+                                     dof_handler, sparsity_pattern);
+   thread_1.join();
+   thread_2.join();
+   hanging_node_constraints.condense (sparsity_pattern);
+ * @endcode
+ *
+ * But what if
+ * your computer has only one processor core, or if we have two but there is
+ * already a different part of the program running in %parallel to the code
+ * above? In that case, we could of course still start new threads, but the
+ * program is not going to run faster since no additional compute resources
+ * are available; rather, the program will run slower since threads have to be
+ * created and destroyed, and the operating system has to schedule threads to
+ * oversubscribed compute resources.
+ *
+ * A better scheme would identify independent tasks and then hand them off to
+ * a scheduler that maps tasks to available compute resources. This way, the
+ * program could, for example, start one thread per processor core and then
+ * let threads work on tasks. Tasks would run to completion, rather than
+ * concurrently, avoiding the overhead of interrupting threads to run a
+ * different thread. In this model, if two processor cores would be available,
+ * tasks 2 and 3 above would run in %parallel; if only one is available, the
+ * scheduler would first completely execute task 2 before doing task 3, or the
+ * other way around. This model is able to execute much more efficiently in
+ * particular if a large number of tasks is available for execution, see for
+ * example the discussion below in section
+ * @ref MTWorkStream "Abstractions for tasks: Work streams". In
+ * essence, tasks are a high-level description of what needs to be done,
+ * whereas threads are a low-level way of implementing how these tasks can be
+ * completed. As in many other instances, being able to use a high-level
+ * description allows to find efficient low-level implementations; in this
+ * vein, it often pays off to use tasks, rather than threads, in a program.
+ *
+ * deal.II does not implement scheduling tasks to threads itself. For this, we
+ * use the <a href="http://www.threadingbuildingblocks.org" target="_top">Threading Building
+ * Blocks (TBB) library</a> for which we provide simple wrappers. TBB
+ * abstracts the details of how to start or stop threads, start tasks on
+ * individual threads, etc, and provides interfaces that are portable across
+ * many different systems.
+ *
+ *
+ * 
+ * @anchor MTUsing
+ * <h3>Using tasks from within deal.II</h3>
+ *
+ * Ideally, the syntax to start tasks (and similarly for threads, for that
+ * matter), would be something like this for the example above:
+ * @code
+   Threads::Task<void>
+     thread
+     = new_task DoFTools::make_hanging_node_constraints (dof_handler,
+                                                         hanging_node_constraints);
+ * @endcode
+ * In other words, we would like to indicate the fact that the function call
+ * should be run on a separate task by simply prefixing the call with a
+ * keyword (such as <code>new_task</code> here, with a similar keyword
+ * <code>new_thread</code> for threads). Prefixing a call would return a
+ * handle for the task that we can use to wait for the tasks's completion and
+ * that we may use to query the return value of the function called (unless it
+ * is void, as it is here).
+ *
+ * Since C++ does not support the creation of new keywords, we have to be a
+ * bit more creative. The way chosen is to introduce a function
+ * <code>new_task</code> that takes as arguments the function to call as well
+ * as the arguments to the call. The <code>new_task</code> function is
+ * overloaded to accomodate starting tasks with functions that take no, one,
+ * two, and up to 9 arguments. In deal.II, these functions live in the Threads
+ * namespace. Consequently, the actual code for what we try to do above looks
+ * like this:
+ * @code
+   Threads::Task<void>
+     thread
+     = Threads::new_task (&DoFTools::make_hanging_node_constraints,
+                          dof_handler,
+                          hanging_node_constraints);
+ * @endcode
+ * Note that DoFTools::make_hanging_node_constraints is a static member
+ * function and so does not need an object of type DoFTools to work on.
+ * (In fact, DoFTools has only static member functions and could as well be
+ * a namespace instead of class; that it is a class at the time of writing
+ * this is mostly a historic relic.)
+ *
+ * Similarly, if we want to call a member function on a different task, we can
+ * do so by specifying the object on which to call the function as first
+ * argument after the function pointer:
+ * @code
+   class C {
+     public:
+       double f(int);
+   };
+
+   int main () {
+     C c;
+
+     // call f(13) as usual, i.e. using the current processor:
+     c.f(13);
+
+     // call f(42) as a separate task, to be scheduled
+     // whenever processor resources become available:
+     Threads::Task<double>
+       task = Threads::new_task (&C::f, c, 42);
+
+     // do something else in between:
+     ...;
+
+     // having finished our other business, wait until the task
+     // above has terminated and get the value returns by c.f(42):
+     double result = task.return_value();
+ * @endcode
+ * Here, note first how we pass the object <code>c</code> (i.e. the
+ * <code>this</code> pointer the function <code>C::f</code> will see) as if it
+ * was the first argument to the function. Secondly, note how we can acquire
+ * the value returned by the function on the separate task by calling
+ * Threads::Task::return_value(). This function implies waiting for the
+ * completion of the task, i.e. the last line is completely equivalent to
+ * @code
+     task.join ();
+     double result = task.return_value();
+ * @endcode
+ *
+ * Note also that it is entirely valid if <code>C::f</code> wants to start
+ * tasks of its own:
+ * @code
+   class C {
+     public:
+       double f(int);
+     private:
+       double f1(int);
+       double f2(int);
+   };
+
+   double C::f (int i) {
+     Threads::Task<double> t1 = Threads::new_task (&C::f1, *this, i);
+     Threads::Task<double> t2 = Threads::new_task (&C::f2, *this, i);
+     return t1.return_value() + t2.return_value();
+   }
+   
+   int main () {
+     C c;
+
+     Threads::Task<double>
+       task = Threads::new_task (&C::f, c, 42);
+
+     // do something else in between:
+     ...;
+
+     double result = task.return_value();
+ * @endcode
+ * Here, we let <code>C::f</code> compute its return value as
+ * <code>c.f1(i)+c.f2(i)</code>. If sufficient CPU resources are available,
+ * then the two parts of the addition as well as the other things in
+ * <code>main()</code> will all run in %parallel. If not, then we will
+ * eventually block at one of the places where the return value is needed,
+ * thereby freeing up the CPU resources necessary to run all those spawned
+ * tasks to completion.
+ *
+ *
+ * In many cases, such as the introductory example of the
+ * <code>setup_dofs</code> function outlined above, one can identify several
+ * independent jobs that can be run as tasks, but will have to wait for all of
+ * them to finish at one point. One can do so by storing the returned object
+ * from all the Threads::new_task() calls, and calling Threads::Task::join()
+ * on each one of them. A simpler way to do this is to put all of these task
+ * objects into a Threads::TaskGroup object and waiting for all of them at
+ * once. The code would then look like this:
+ * @code
+   dof_handler.distribute_dofs (fe);
+
+   Threads::TaskGroup<void> task_group;
+   task_group += Threads::new_task (&DoFTools::make_hanging_node_constraints,
+                                    dof_handler, hanging_node_constraints);
+   task_group += Threads::new_task (&DoFTools::make_sparsity_pattern,
+                                    dof_handler, sparsity_pattern);
+   task_group.join_all ();
+   hanging_node_constraints.condense (sparsity_pattern);
+ * @endcode
+ *
+ *
+ * @anchor MTHow
+ * <h3>How scheduling tasks works and when task-based programming is not efficient</h3>
+ *
+ * The exact details of how tasks are scheduled to run are %internal to the
+ * Threading Building Blocks (TBB) library that deal.II uses for tasks. The
+ * documentation of TBB gives a detailed description of how tasks are
+ * scheduled to threads but is rather quiet on how many threads are actually
+ * used. However, a reasonable guess is probably to assume that TBB creates as
+ * many threads as there are processor cores on your system. This way, it is
+ * able to fully utilize the entire system, without having too many threads
+ * that the operating system will then have to interrupt regularly so that
+ * other threads can run on the available processor cores.
+ *
+ * The point then is that the TBB scheduler takes tasks and lets threads
+ * execute them. %Threads execute tasks completely, i.e. the TBB scheduler does
+ * not interrupt a task half way through to make some halfway progress with
+ * another task. This makes sure that caches are always hot, for example, and
+ * avoids the overhead of preemptive interrupts.
+ *
+ * The downside is that the CPU cores are only fully utilized if the threads
+ * are actually doing something, and that means that (i) there must be enough
+ * tasks available, and (ii) these tasks are actually doing something. Note
+ * that both conditions must be met; in particular, this means that CPU cores
+ * are underutilized if we have identified a sufficient number of tasks but if
+ * some of them twiddle thumbs, for example because a task is writing data to
+ * disk (a process where the CPU frequently has to wait for the disk to
+ * complete a transaction) or is waiting for input. Other cases are where
+ * tasks block on other external events, for example synchronising with other
+ * tasks or threads through a mutex. In such cases, the scheduler would let a
+ * task run on a thread, but doesn't notice that that thread doesn't fully
+ * utilize the CPU core.
+ *
+ * In cases like these, it <i>does</i> make sense to create a new thread (see
+ * @ref MTThreads "Thread-based parallelism" below) that the operating system
+ * can put on hold while they are waiting for something external, and let a
+ * different thread (for example one running a task scheduled by TBB) use the
+ * CPU at the same time.
+ *
+ *
+ * @anchor MTSimpleLoops
+ * <h3>Abstractions for tasks: Simple loops</h3>
+ *
+ * Some loops execute bodies on data that is completely independent
+ * and that can therefore be executed in %parallel. Rather than a
+ * priori split the loop into a fixed number of chunks and executing
+ * them on tasks or threads, the TBB library uses the following
+ * concept: the range over which the loop iterates is split into a
+ * certain number of sub-ranges (for example two or three times as
+ * many as there are CPU cores) and are equally distributed among
+ * threads; threads then execute sub-ranges and, if they are done with
+ * their work, steal entire or parts of sub-ranges from other threads
+ * to keep busy. This way, work is load-balanced even if not every
+ * loop iteration takes equally much work, or if some of the CPUs fall
+ * behind because the operating system interrupted it for some other
+ * work.
+ *
+ * The TBB library primitives for this are a bit clumsy so deal.II has
+ * wrapper routines for the most frequently use operations. The
+ * simplest one is akin to the what the std::transform does: it takes
+ * one or more ranges of input operators, one output iterator, and a
+ * function object. A typical implementation of std::transform would
+ * look like this:
+ * @code
+     template <typename InputIterator1, typename InputIterator,
+               typename OutputIterator, typename FunctionObject>
+     void transform (const InputIterator1 &begin_in_1,
+                     const InputIterator1 &end_in_1,
+                     const InputIterator2 &begin_in_2,
+                     const OutputIterator &begin_out,
+                    FunctionObject       &function)
+     {
+       InputIterator1 in_1 = begin_in_1;
+       InputIterator2 in_2 = begin_in_2;
+       OutputIterator out  = begin_out;
+
+       for (; in_1 != end_in_1; ++in_1, ++in_2, ++out)
+         *out = function(*in_1, *in_2);
+     }
+ * @endcode
+ *
+ * In many cases, <code>function</code> has no state, and so we can
+ * split this loop into several sub-ranges as explained
+ * above. Consequently, deal.II has a set of functions
+ * parallel::transform that look like the one above but that do their
+ * work in %parallel (there are several versions with one, two, and
+ * more input iterators for function objects that take one, two, or
+ * more arguments). The only difference in calling these functions is
+ * that they take an additional last argument that denotes the minimum
+ * size of sub-ranges of <code>[begin_in_1,end_in_1)</code>; it should
+ * be big enough so that we don't spend more time on scheduling
+ * sub-ranges to processors but small enough that processors can be
+ * efficiently load balanced. A rule of thumb appears to be that a
+ * sub-range is too small if it takes less than 2000 instructions to
+ * execute it.
+ *
+ * An example of how to use these functions are vector operations like
+ * the addition in $z = x+y$ where all three objects are of type Vector:
+ * @code
+     parallel::transform (x.begin(), x.end(),
+                          y.begin(),
+                         z.begin(),
+                         (boost::lambda::_1 + boost::lambda::_2),
+                         1000);
+ * @endcode
+ *
+ * In this example, we used the <a
+ * href="http://www.boost.org/doc/libs/1_37_0/doc/html/lambda.html">Boost
+ * Lambda</a> library to construct, on the fly, a function object that
+ * takes two arguments and returns the sum of the two. This is exactly
+ * what we needed when we want to add the individual elements of
+ * vectors $x$ and $y$ and write the sum of the two into the elements
+ * of $z$. Because of the way Boost Lambda is written, the function
+ * object that we get here is completely known to the compiler and
+ * when it expands the loop that results from parallel::transform will
+ * be as if we had written the loop in its obvious form:
+ * @code
+       InputIterator1 in_1 = x.begin();
+       InputIterator2 in_2 = y.begin();
+       OutputIterator out  = z.begin();
+
+       for (; in_1 != x.end(); ++in_1, ++in_2, ++out)
+         *out = *in_1 + *in_2;
+ * @endcode
+ * The next C++ standard will contain a more elegant way to achieve the
+ * same effect shown above using the Boost Lambda library, through a
+ * mechanism known as <i>lambda expressions</i> and <i>closures</i>.
+ *
+ * Note also that we have made sure that no CPU ever gets a chunk of
+ * the whole loop that is smaller than 1000 iterations (unless the
+ * whole range is smaller).
+ *
+ *
+ * @anchor MTComplexLoops
+ * <h3>Abstractions for tasks: More complex loops</h3>
+ *
+ * The scheme shown in the previous section is effective if the
+ * operation done in each iteration is such that it does not require
+ * significant setup costs and can be inlined by the compiler. Lambda
+ * expressions are exactly of this kind because the compiler knows
+ * everything about the lambda expression and can inline it, thereby
+ * eliminating the overhead of calling an external function. However,
+ * there are cases where it is inefficient to call some object or
+ * function within each iteration.
+ *
+ * An example for this case is sparse matrix-vector multiplication. If you
+ * know how data is stored in compressed row format like in the SparseMatrix
+ * class, then a matrix-vector product function looks like this:
+ * @code
+    void SparseMatrix::vmult (const Vector &src,
+                              Vector       &dst) const
+    {
+      const double       *val_ptr    = &values[0];
+      const unsigned int *colnum_ptr = &colnums[0];
+      Vector::iterator dst_ptr = dst.begin();
+
+      for (unsigned int row=0; row<n_rows; ++row, ++dst_ptr)
+       {
+         double s = 0.;
+         const double *const val_end_of_row = &values[rowstart[row+1]];
+         while (val_ptr != val_end_of_row)
+           s += *val_ptr++ * src(*colnum_ptr++);
+         *dst_ptr = s;
+       }
+    }
+ * @endcode
+ * Inside the for loop, we compute the dot product of a single row of the
+ * matrix with the right hand side vector <code>src</code> and write it into
+ * the corresponding element of the <code>dst</code> vector. The code is made
+ * more efficient by utilizing that the elements of the <i>next</i> row follow
+ * the ones of the current row <i>immediately</i>, i.e. at the beginning of
+ * the loop body we do not have to re-set the pointers that point to the
+ * values and column %numbers of each row.
+ *
+ * Using the parallel::transform function above, we could in principle write
+ * this code as follows:
+ * @code
+    void SparseMatrix::vmult (const Vector     &src,
+                              Vector           &dst,
+                             Vector::iterator &dst_row) const
+    {
+      const unsigned int  row = (dst_row - dst.begin());
+      
+      const double       *val_ptr    = &values[rowstart[row]];
+      const unsigned int *colnum_ptr = &colnums[rowstart[row]];
+
+      double s = 0.;
+      const double *const val_end_of_row = &values[rowstart[row+1]];
+      while (val_ptr != val_end_of_row)
+       s += *val_ptr++ * src(*colnum_ptr++);
+      *dst_row = s;
+    }
+
+    void SparseMatrix::vmult (const Vector &src,
+                              Vector       &dst) const
+    {
+      parallel::transform (dst.begin(), dst.end(),
+                           boost::bind (&SparseMatrix::vmult_one_row,
+                                       this,
+                                       boost::cref(src),
+                                       boost::ref(dst),
+                                       _1),
+                          200);
+    }
+ * @endcode
+ * Note how we use <a
+ * href="http://www.boost.org/doc/libs/1_37_0/libs/bind/bind.html">boost::bind</a>
+ * to <i>bind</i> certain arguments to the <code>vmult_one_row</code>
+ * function, leaving one argument open and thus allowing the
+ * parallel::transform function to consider the passed function argument as
+ * unary. Also note that we need to make the source and destination vectors as
+ * (const) references to prevent boost::bind from passing them by value
+ * (implying a copy for <code>src</code> and writing the result into a
+ * temporary copy of <code>dst</code>, neither of which is what we desired).
+ * Finally, notice the grainsize of a minimum of 200 rows of a matrix that
+ * should be processed by an individual CPU core.
+ *
+ * The point is that while this is correct, it is not efficient: we have to
+ * set up the <code>row, val_ptr, colnum_ptr</code> variables in each
+ * iteration of the loop. Furthermore, since now the function object to be
+ * called on each row is not a simple Boost Lambda expression any more, there
+ * is an implied function call including argument passing in each iteration of
+ * the loop.
+ *
+ * A more efficient way is to let TBB split the original range into
+ * sub-ranges, and then call a target function not on each individual element
+ * of the loop, but on the entire range. This is facilitated by the
+ * parallel::apply_to_subranges function:
+ * @code
+    void
+    SparseMatrix::vmult_on_subrange (const unsigned int  begin_row,
+                                    const unsigned int  end_row,
+                                    const Vector     &src,
+                                    Vector           &dst)
+    {
+      const double       *val_ptr    = &values[rowstart[begin_row]];
+      const unsigned int *colnum_ptr = &colnums[rowstart[begin_row]];
+      Vector::iterator dst_ptr = dst.begin() + begin_row;
+
+      for (unsigned int row=begin_row; row<end_row; ++row, ++dst_ptr)
+       {
+         double s = 0.;
+         const double *const val_end_of_row = &values[rowstart[row+1]];
+         while (val_ptr != val_end_of_row)
+           s += *val_ptr++ * src(*colnum_ptr++);
+         *dst_ptr = s;
+       }
+    }
+
+    void SparseMatrix::vmult (const Vector &src,
+                              Vector       &dst) const
+    {
+       parallel::apply_to_subranges (0, n_rows(),
+                                    boost::bind (vmult_on_subrange,
+                                                 this,
+                                                 _1, _2,
+                                                 boost::cref(src),
+                                                 boost::ref(dst)),
+                                    200);
+    } 
+ * @endcode
+ * Here, we call the <code>vmult_on_subrange</code> function on sub-ranges
+ * of at least 200 elements each, so that the initial setup cost can amortize.
+ *
+ * A related operation is when the loops over elements each produce a
+ * result that must then be accumulated (other reduction operations
+ * than addition of numbers would work as well). An example is to form
+ * the matrix norm $x^T M x$ (it really is only a norm if $M$ is
+ * positive definite, but let's assume for a moment that it is). A
+ * sequential implementation would look like this for sparse matrices:
+ * @code
+    double SparseMatrix::mat_norm (const Vector &x) const
+    {
+      const double       *val_ptr    = &values[0];
+      const unsigned int *colnum_ptr = &colnums[0];
+
+      double norm_sqr = 0;
+      
+      for (unsigned int row=0; row<n_rows; ++row, ++dst_ptr)
+       {
+         double s = 0.;
+         const double *const val_end_of_row = &values[rowstart[row+1]];
+         while (val_ptr != val_end_of_row)
+           s += *val_ptr++ * x(*colnum_ptr++);
+         norm_sqr += x(row) * s;
+       }
+
+      return std::sqrt (norm_sqr);
+    }
+ * @endcode
+ *
+ * It would be nice if we could split this operation over several
+ * sub-ranges of rows, each of which compute their part of the square
+ * of the norm, add results together from the various sub-ranges, and
+ * then take the square root of the result. This is what the
+ * parallel::accumulate_from_subranges function does (note that you
+ * have to specify the result type as a template argument and that, as
+ * usual, the minimumum number of elements of the outer loop that can
+ * be scheduled on a single CPU core is given as the last argument):
+ * @code
+    double
+    SparseMatrix::mat_norm_sqr_on_subrange (const unsigned int  begin_row,
+                                           const unsigned int  end_row,
+                                           const Vector     &x)
+    {
+      const double       *val_ptr    = &values[rowstart[begin_row]];
+      const unsigned int *colnum_ptr = &colnums[rowstart[begin_row]];
+      Vector::iterator dst_ptr = dst.begin() + begin_row;
+
+      double norm_sqr = 0;
+      
+      for (unsigned int row=begin_row; row<end_row; ++row, ++dst_ptr)
+       {
+         double s = 0.;
+         const double *const val_end_of_row = &values[rowstart[row+1]];
+         while (val_ptr != val_end_of_row)
+           s += *val_ptr++ * x(*colnum_ptr++);
+         norm_sqr += x(row) * s;
+       }
+
+      return norm_sqr;
+    }
+
+    double SparseMatrix::mat_norm (const Vector &x) const
+    {
+      return
+        std::sqrt
+       (parallel::accumulate_from_subranges (0, n_rows(),
+                                             boost::bind (mat_norm_sqr_on_subrange,
+                                                          this,
+                                                          _1, _2,
+                                                          boost::cref(x)),
+                                             200));
+    } 
+ * @endcode
+ *
+ *
+ * @anchor MTWorkStream
+ * <h3>Abstractions for tasks: Work streams</h3>
+ *
+ * In the examples shown in the introduction we had identified a
+ * number of functions that can be run as independent tasks. Ideally,
+ * this number of tasks is larger than the number of CPU cores (to
+ * keep them busy) but is also not exceedingly huge (so as not to
+ * inundate the scheduler with millions of tasks that will then have
+ * to be distributed to 2 or 4 cores, for example). There are,
+ * however, cases where we have many thousands or even millions of
+ * relatively independent jobs: for example, assembling local
+ * contributions to the global linear system on each cell of a mesh;
+ * evaluating an error estimator on each cell; or postprocessing on
+ * each cell computed data for output fall into this class.
+ *
+ * Code like this could then be written like this:
+ * @code
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell)
+   { ... }
+   
+   template <int dim>
+   void MyClass<dim>::assemble_system ()
+   {
+     Threads::TaskGroup<void> task_group;
+     for (typename DoFHandler<dim>::active_cell_iterator
+            cell = dof_handler.begin_active();
+         cell != dof_handler.end(); ++cell)
+       task_group += Threads::new_task (&MyClass<dim>::assemble_on_one_cell,
+                                        *this,
+                                       cell);
+     task_group.join_all ();
+   }
+ * @endcode
+ * On a big mesh, with maybe a million cells, this would create a massive
+ * number of tasks; while it would keep all CPU cores busy for a while, the
+ * overhead of first creating so many tasks, scheduling them, and then waiting
+ * for them would probably not lead to efficient code. A better strategy would
+ * be if the scheduler could somehow indicate that it has available resources,
+ * at which point we would feed it another newly created task, and we would do
+ * so until we run out of tasks and the ones that were created have been
+ * worked on.
+ *
+ * This is essentially what the WorkStream class does: You give it an iterator
+ * range from which it can draw objects to work on (in the above case it is
+ * the interval given by <code>dof_handler.begin_active()</code> to
+ * <code>dof_handler.end()</code>), and a function that would do the work on
+ * each item (the function <code>MyClass::assemble_on_one_cell</code>)
+ * together with an object if it is a member function.
+ *
+ * Essentially, the way the <code>MyClass::assemble_system</code>
+ * function could be written then is like this (note that this is not quite
+ * the correct syntax, as will be described below):
+ * @code
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell)
+   { ... }
+   
+   template <int dim>
+   void MyClass<dim>::assemble_system ()
+   {
+     WorkStream work_stream;
+     work_stream.run (dof_handler.begin_active(),
+                     dof_handler.end(),
+                     *this,
+                     &MyClass<dim>::assemble_on_one_cell);
+   }
+ * @endcode
+ *
+ * There are at least three problems with this, however:
+ *<ul>
+ *<li>First, let us take a look at how the <code>MyClass::assemble_on_one_cell</code>
+ *   function likely looks:
+ * @code
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell)
+   {
+     FEValues<dim> fe_values (...);
+     FullMatrix<double> cell_matrix (...);
+     Vector<double>     cell_rhs (...);
+
+     // assemble local contributions
+     fe_values.reinit (cell);
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         for (unsigned int q=0; q<fe_values.n_quadrature_points; ++q)
+          cell_matrix(i,j) += ...;
+     ...same for cell_rhs...
+
+     // now copy results into global system
+     std::vector<unsigned int> dof_indices (...);
+     cell->get_dof_indices (dof_indices);
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         system_matrix.add (dof_indices[i], dof_indices[j],
+                           cell_matrix(i,j));
+     ...same for rhs...
+   }
+ * @endcode
+ *   The problem here is that several tasks, each running
+ *   <code>MyClass::assemble_on_one_cell</code>, could potentially try
+ *   to write into the object <code>MyClass::system_matrix</code> <i>at
+ *   the same time</i>. This could be avoided by explicit synchronisation
+ *   using a Threads::Mutex, for example, and would look like this:
+ * @code
+     // now copy results into global system
+     std::vector<unsigned int> dof_indices (...);
+     cell->get_dof_indices (dof_indices);
+
+     static Threads::Mutex mutex;
+     mutex.acquire ();
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         system_matrix.add (dof_indices[i], dof_indices[j],
+                           cell_matrix(i,j));
+     ...same for rhs...
+     mutex.release ();
+   }
+ * @endcode
+ *   By making the mutex a static variable, it exists only once globally
+ *   (i.e. once for all tasks that may be running in %parallel) and only one of
+ *   the tasks can enter the region protected by the acquire/release calls on
+ *   the mutex. As an aside, a better way to write this code would be like
+ *   this, ensuring that the mutex is released even in case an exception is
+ *   thrown, and without the need to remember to write the call to
+ *   Threads::Mutex::release():
+ * @code
+     // now copy results into global system
+     static Threads::Mutex mutex;
+     Threads::Mutex::ScopedLock lock (mutex);
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         system_matrix.add (dof_indices[i], dof_indices[j],
+                           cell_matrix(i,j));
+     ...same for rhs...
+   }
+ * @endcode
+ *   Here, the mutex remains locked from the time the ScopedLock is created to
+ *   where it is destroyed, at the end of the code block.
+ *
+ *   Note that although we now avoid the race condition that multiple threads
+ *   could be writing to the same object, this code is not very efficient:
+ *   mutexes are expensive on multicore machines, and we also block threads
+ *   some of the time which is inefficient with tasks as explained above in
+ *   the section on
+ *   @ref MTHow "How scheduling tasks works and when task-based programming is not efficient".
+ *
+ *<li>A second correctness problem is that even if we do lock the global matrix
+ *   and right hand side objects using a mutex, we do so in a more or less
+ *   random order: while tasks are created in the order in which we traverse
+ *   cells normally, there is no guarantee that by the time we get to the
+ *   point where we want to copy the local into the global contributions the
+ *   order is still as if we computed things sequentially. In other words, it
+ *   may happen that we add the contributions of cell 1 before those of cell
+ *   0. That may seem harmless because addition is commutative, but in fact it
+ *   is not if done in floating point arithmetic: $a+b+c \neq a+c+b$ -- take
+ *   for example $a=1, b=-1, c=10^{-20}$ (because $1+10^{-20}=1$ in floating
+ *   point arithmetic, using double precision).
+ *
+ *   As a consequence, the exact values that end up in the global matrix and
+ *   right hand side will be close but may differ by amounts close to
+ *   round-off depending on the order in which tasks happened to finish their
+ *   job. That's not a desirable outcome, since results will not be
+ *   reproducible this way.
+ *
+ *   As a consequence, the way the WorkStream class is designed is to use two
+ *   functions: the <code>MyClass::assemble_on_one_cell</code> computes the
+ *   local contributions and stores it somewhere (we'll get to that next), and
+ *   a second function, say <code>MyClass::copy_local_to_global</code>, that
+ *   copies the results computed on each cell into the global objects. The
+ *   trick implemented in the WorkStream class is that (i) the
+ *   <code>MyClass::copy_local_to_global</code> never runs more than once in
+ *   %parallel, so we do not need to synchronise execution through a mutex, and
+ *   (ii) it runs in exactly the same order on cells as they appear in the
+ *   iterator range, i.e. we add elements into the global matrix the same way
+ *   <i>every time, independently of when the computation of these element
+ *   finishes</i>.
+ *
+ *   We now only have to discuss how the
+ *   <code>MyClass::assemble_on_one_cell</code> communicates to
+ *   <code>MyClass::copy_local_to_global</code> what it has computed. The way
+ *   this is done is to use an object that holds all temporary data:
+ * @code
+   struct PerTaskData {
+     FullMatrix<double>        cell_matrix;
+     Vector<double>            cell_rhs;
+     std::vector<unsigned int> dof_indices;
+   }
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                            PerTaskData &data)
+   {
+     FEValues<dim> fe_values (...);
+
+     data.cell_matrix = 0;
+     data.cell_rhs    = 0;
+     
+     // assemble local contributions
+     fe_values.reinit (cell);
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         for (unsigned int q=0; q<fe_values.n_quadrature_points; ++q)
+          data.cell_matrix(i,j) += ...;
+     ...same for cell_rhs...
+
+     cell->get_dof_indices (data.dof_indices);
+   }
+
+   template <int dim>
+   void MyClass<dim>::copy_local_to_global (const PerTaskData &data)
+   {
+     for (unsigned int i=0; i<fe.dofs_per_cell; ++i)
+       for (unsigned int j=0; j<fe.dofs_per_cell; ++j)
+         system_matrix.add (data.dof_indices[i], data.dof_indices[j],
+                           data.cell_matrix(i,j));
+     ...same for rhs...
+   }
+   
+   template <int dim>
+   void MyClass<dim>::assemble_system ()
+   {
+     PerTaskData per_task_data;
+     ...initialize members of per_task_data to the correct sizes...
+     
+     WorkStream work_stream;
+     work_stream.run (dof_handler.begin_active(),
+                     dof_handler.end(),
+                     *this,
+                     &MyClass<dim>::assemble_on_one_cell,
+                     &MyClass<dim>::copy_local_to_global,
+                     per_task_data);
+   }
+ * @endcode 
+ *
+ *   The way this works is that we create a sample <code>per_task_data</code>
+ *   object that the work stream object will replicate once per task that runs
+ *   in %parallel. For each task, this object will be passed first to one of
+ *   possibly several instances of <code>MyClass::assemble_on_one_cell</code>
+ *   running in %parallel which fills it with the data obtained on a single
+ *   cell, and then to a sequentially running
+ *   <code>MyClass::copy_local_to_global</code> that copies data into the
+ *   global object. In practice, of course, we will not generate millions of
+ *   <code>per_task_data</code> objects if we have millions of cells; rather,
+ *   we recycle these objects after they have been used by
+ *   <code>MyClass::copy_local_to_global</code> and feed them back into
+ *   another instance of <code>MyClass::assemble_on_one_cell</code>; this
+ *   means that the number of such objects we actually do create is a small
+ *   multiple of the number of threads the scheduler uses, which is typically
+ *   about as many as there are CPU cores on a system.
+ *
+ * <li>The last issue that is worth addressing is that the way we wrote the
+ *   <code>MyClass::assemble_on_one_cell</code> function above, we create and
+ *   destroy an FEValues object every time the function is called, i.e. once
+ *   for each cell in the triangulation. That's an immensely expensive
+ *   operation because the FEValues class tries to do a lot of work in its
+ *   constructor in an attempt to reduce the number of operations we have to
+ *   do on each cell (i.e. it increases the constant in the ${\cal O}(1)$
+ *   effort to initialize such an object in order to reduce the constant in
+ *   the ${\cal O}(N)$ operations to call FEValues::reinit on the $N$ cells of
+ *   a triangulation). Creating and destroying an FEValues object on each cell
+ *   invalidates this effort.
+ *
+ *   The way to avoid this is to put the FEValues object into a second
+ *   structure that will hold scratch data, and initialize it in the
+ *   constructor: 
+ * @code
+   struct PerTaskData {
+     FullMatrix<double>        cell_matrix;
+     Vector<double>            cell_rhs;
+     std::vector<unsigned int> dof_indices;
+
+     PerTaskData (const FiniteElement<dim> &fe)
+               :
+               cell_matrix (fe.dofs_per_cell, fe.dofs_per_cell),
+               cell_rhs (fe.dofs_per_cell),
+               dof_indices (fe.dofs_per_cell)
+       {}
+   }
+
+   struct ScratchData {
+     FEValues<dim>             fe_values;
+
+     ScratchData (const FiniteElement<dim> &fe,
+                  const Quadrature<dim>    &quadrature,
+                  const UpdateFlags         update_flags)
+               :
+               fe_values (fe, quadrature, update_flags)
+       {}
+   }
+ * @endcode
+ * and then use this FEValues object in the assemble function:
+ * @code
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                            ScratchData &scratch,
+                                           PerTaskData &data)
+   {
+     scratch.fe_values.reinit (cell);
+     ...
+   }
+ * @endcode
+ *   The same approach, putting things into the <code>ScratchData</code>
+ *   data structure should be used for everything that is expensive to
+ *   construct. This holds, in particular, for everything that needs to
+ *   allocate memory upon construction; for example, if the values of a
+ *   function need to be evaluated at quadrature points, then this is
+ *   expensive:
+ * @code
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                            ScratchData &scratch,
+                                           PerTaskData &data)
+   {
+     std::vector<double> rhs_values (fe_values.n_quadrature_points);
+     rhs_function.value_list (data.fe_values.get_quadrature_points,
+                              rhs_values)
+     ...
+   }
+ * @endcode
+ * whereas this is a much cheaper way:
+ * @code
+   struct ScratchData {
+     std::vector<double>       rhs_values;
+     FEValues<dim>             fe_values;
+
+     ScratchData (const FiniteElement<dim> &fe,
+                  const Quadrature<dim>    &quadrature,
+                  const UpdateFlags         update_flags)
+               :
+               rhs_values (quadrature.n_quadrature_points),
+               fe_values (fe, quadrature, update_flags)
+       {}
+   }
+
+   template <int dim>
+   void MyClass<dim>::assemble_on_one_cell (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                            ScratchData &scratch,
+                                           PerTaskData &data)
+   {
+     rhs_function.value_list (scratch.fe_values.get_quadrature_points,
+                              scratch.rhs_values)
+     ...
+   }
+ * @endcode
+ *
+ * </ul>
+ *
+ * As a final point: What if, for some reason, my assembler and copier
+ * function do not match the above signature with three and one argument,
+ * respectively? That's not a problem either. The WorkStream class offers two
+ * versions of the WorkStream::run() function: one that takes an object and
+ * the addresses of two member functions, and one that simply takes two
+ * function objects that can be called with three and one argument,
+ * respectively. So, in other words, the following two calls are exactly
+ * identical:
+ * @code
+     work_stream.run (dof_handler.begin_active(),
+                     dof_handler.end(),
+                     *this,
+                     &MyClass<dim>::assemble_on_one_cell,
+                     &MyClass<dim>::copy_local_to_global,
+                     per_task_data);
+     // ...is the same as:                   
+     work_stream.run (dof_handler.begin_active(),
+                     dof_handler.end(),
+                     boost::bind(&MyClass<dim>::assemble_on_one_cell, *this, _1, _2, _3),
+                     boost::bind(&MyClass<dim>::copy_local_to_global, *this, _1),
+                     per_task_data);
+ * @endcode
+ * Note how <code>boost::bind</code> produces a function object that takes three
+ * arguments by binding the member function to the <code>*this</code>
+ * object. <code>_1, _2</code> and <code>_3</code> are placeholders for the first,
+ * second and third argument that can be specified later on. In other words, for
+ * example if <code>p</code> is the result of the first call to
+ * <code>boost::bind</code>, then the call <code>p(cell, scratch_data,
+ * per_task_data)</code> will result in executing
+ * <code>this-@>assemble_on_one_cell (cell, scratch_data, per_task_data)</code>,
+ * i.e. <code>boost::bind</code> has bound the object to the function pointer
+ * but left the three arguments open for later.
+ *
+ * Similarly, let us assume that <code>MyClass::assemble_on_one_cell</code>
+ * has the following signature in a solver of nonlinear, time-dependent problem:
+ * @code
+   template <int dim>
+   void
+   MyClass<dim>::assemble_on_one_cell (const Vector<double> &linearization_point,
+                                       const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                       ScratchData &scratch,
+                                       PerTaskData &data,
+                                      const double current_time)
+   { ... }
+ * @endcode
+ * Because WorkStream expects to be able to call the worker function with
+ * just three arguments, the first of which is the iterator and the second
+ * and third the ScratchData and PerTaskData objects, we need to pass the following
+ * to it:
+ * @code
+     work_stream.run (dof_handler.begin_active(),
+                     dof_handler.end(),
+                     boost::bind(&MyClass<dim>::assemble_on_one_cell,
+                                 *this,
+                                 current_solution,
+                                 _1,
+                                 _2,
+                                 _3,
+                                 previous_time+time_step),
+                     boost::bind(&MyClass<dim>::copy_local_to_global,
+                                 *this, _1),
+                     per_task_data);
+ * @endcode
+ * Here, we bind the object, the linearization point argument, and the
+ * current time argument to the function before we have it off to
+ * WorkStream::run(). WorkStream::run() will then simply call the
+ * function with the cell and scratch and per task objects which will be filled
+ * in at the positions indicated by <code>_1, _2</code> and <code>_3</code>.
+ *
+ *
+ * @anchor MTThreads
+ * <h3>Thread-based parallelism</h3>
+ *
+ * Even though tasks are a higher-level way to describe things, there are
+ * cases where they are poorly suited to a task. The main reason for not using
+ * tasks even for computations that are independent are listed in the section
+ * on
+ * @ref MTHow "How scheduling tasks works and when task-based programming is not efficient"
+ * above. Primarily, jobs that are not able to fully utilize are bad fits for tasks.
+ *
+ * In a case like this, you can resort to explicitly start threads, rather
+ * than tasks, using pretty much the same syntax as above. For example, if you
+ * had a function in your application that generates graphical output and then
+ * estimates the error to refine the mesh for the next iteration of an
+ * adaptive mesh scheme, it could look like this:
+ * @code
+   template <int dim>
+   void MyClass<dim>::output_and_estimate_error () const
+   {
+     DataOut<dim> data_out;
+     data_out.attach_dof_handler (dof_handler);
+     data_out.add_data_vector (solution, "solution");
+     data_out.build_patches ();
+
+     std::ofstream output ("solution.vtk");
+     
+     Threads::Thread<void>
+       thread = Threads::new_thread (&DataOut<dim>::write_vtk, data_out, output);
+
+     Vector<float> error_per_cell (triangulation.n_active_cells());
+     KellyErrorEstimator<dim>::estimate (dof_handler,
+                                       QGauss<dim-1>(3),
+                                       typename FunctionMap<dim>::type(),
+                                       solution,
+                                       estimated_error_per_cell);
+     thread.join ();                                   
+ * @endcode
+ *
+ * Here, Threads::new_thread starts the given function that writes to the
+ * output file on a new thread that can run in %parallel to everything
+ * else. Note that this function is actually pretty well parallelized: both
+ * DataOut::build_patches() and KellyErrorEstimator::estimate() already use
+ * WorkStream and will therefore utilize pretty much all available compute
+ * resources. In %parallel to the KellyErrorEstimator::estimate() function, the
+ * DataOut::write_vtk() function will run on a %parallel thread, independent of
+ * the scheduler that takes care of the tasks, but that is not a problem
+ * because writing lots of data to a file is not something that will keep a
+ * CPU very busy.
+ *
+ * Creating threads works pretty much the same way as tasks, i.e. you can wait
+ * for the termination of a thread using Threads::Thread::join(), query the
+ * return value of a finished thread using Threads::Thread::return_value(),
+ * and you can group threads into a Threads::ThreadGroup object and wait for
+ * all of them to finish.
  */
diff --git a/deal.II/doc/news/changes-tbb-branch.h b/deal.II/doc/news/changes-tbb-branch.h
new file mode 100644 (file)
index 0000000..6e9ea86
--- /dev/null
@@ -0,0 +1,93 @@
+   <li>
+   <p> 
+   Changed: The two DataOut::build_patches, DataOutFaces::build_patches, and
+   DataOutRotation::build_patches functions have lost the argument
+   that indicated the number of threads with which they should build the
+   intermediate representation. This is something that now happens
+   transparently in the background and doesn't need caller input any more.
+   <br>
+   (WB 2008/12/16)
+   </p>
+
+<li>
+   <p> 
+   Changed: Previously, one had to give the two flags
+   <code>--enable-multithreading --with-multithreading</code> to
+   <code>./configure</code> to enable thread usage throughout the library.
+   This has now been simplified: only the flag <code>--enable-threads</code>
+   is now necessary. Furthermore, since most current machines have multiple
+   cores these days, the default is now to use threads. This can be switched
+   off using <code>--disable-threads</code>, however.
+   <br>
+   (WB 2008/09/29)
+   </p>
+   <li>
+   <p>
+   New: As a primary means of parallelizing programs, deal.II now uses
+   a task-based, rather than thread-based approach, in which one
+   uses a high-level description of <i>what</i> needs to be done,
+   rather than how these jobs have to be mapped onto threads. We then
+   use the <a href="http://www.threadingbuildingblocks.org">Threading
+   Building Blocks (TBB) library</a> to schedule tasks onto available
+   hardware resources. This new scheme of describing parallism and
+   various abstractions to make programming in this framework easier
+   are described in great detail in the
+   @ref threads "Parallel computing with multiple processors" module.
+   In addition, most of the parallelism already used within deal.II
+   has been converted to use tasks, rather than threads, and so have
+   some of the tutorial programs.
+   <br>
+   (WB 2009/01/09)
+   </p>
+   </li>
+   <li>
+   <p>
+   Changed: The support for threading has been completely re-written. In
+   particular, the Threads::spawn functions have been deprecated, and
+   new functions Threads::new_threads have been introduced.
+   Threading is now discussed in a lot of detail in the
+   @ref threads "Parallel computing with multiple processors" module.
+   <br>
+   (WB 2009/01/09)
+   </p>
+   </li>
+   
+  <li>
+   <p>
+   Fixed: The DoFRenumbering::component_wise function for MGDoFHandler objects
+   did a few things in parallel that weren't thread-safe. This is now fixed.
+   <br>
+   (WB, 2009/01/20)
+</p>
+   <li>
+   <p>
+   Changed: The KellyErrorEstimator::estimate functions had a parameter
+   that indicates the number of threads to be used in the computation.
+   This parameter continues to exist for compatibility, but is now ignored.
+   Rather, the number of threads is determined automatically by scheduling
+   the requested computations on available compute resources.
+   <br>
+   (WB, 2008/12/29)
+   </p>
+   <li>
+   <p>
+   New: The new function internal::hp::FEValuesBase::get_fe_collection function
+   allows to query the finite element collection currently in used in an hp::FEValues,
+   hp::FEFaceValues, or hp::FESubfaceValues object.
+   <br>
+   (WB 2008/09/30)
+   </p> 
+   <li>
+   <p>
+   New: The new function FEValuesBase::get_update_flags allows to query
+   the update flags that are currently set on an FEValues, FEFaceValues, or
+   FESubfaceValues object.
+   <br>
+   (WB 2008/09/29)
+   </p> 
index 371601f5ee93bd8157b41de9524232c58bebf118..38db95c41ba324a3bbaef062eed64ff6efc6a749 100644 (file)
 
     <li>
       <p>
-      <code>--enable-multithreading --with-multithreading</code>: The
-      first of these flags instructs the compiler to generate code
-      that can be run from multiple threads. The second switches on
-      code in the library that actually creates multiple threads for
-      some tasks, making programs significantly faster on machines
-      with multiple cores or processors.
+      <code>--enable-threads</code>: This flag indicates that those
+      parts of the library that support this compute in parallel,
+      using multiple threads, making programs significantly faster on
+      machines with multiple cores or processors. The default is to
+      use multiple threads since most machines today have several
+      processor cores. If this is not desired,
+      use <code>--disable-threads</code>.
 
     <li>
       <p>
index 4b9d28f851795a4711056ec54c34c48062540a21..5387fa75b2956c56346b3a4a26416a11195da322 100644 (file)
@@ -1134,13 +1134,15 @@ namespace LaplaceSolver
   }
   
 
-                                  // The following function assembles
-                                  // matrix and right hand side of
-                                  // the linear system to be solved
-                                  // in each step. It goes along the
-                                  // same lines as used in previous
+                                  // The following function assembles matrix
+                                  // and right hand side of the linear system
+                                  // to be solved in each step. It goes along
+                                  // the same lines as used in previous
                                   // examples, so we explain it only
-                                  // briefly:
+                                  // briefly. Note that we do a number of
+                                  // things in parallel, a process described
+                                  // in more detail in the @ref threads
+                                  // module.
   template <int dim>
   void
   Solver<dim>::assemble_linear_system (LinearSystem &linear_system)
@@ -1183,13 +1185,14 @@ namespace LaplaceSolver
     Threads::ThreadMutex mutex;
     Threads::ThreadGroup<> threads;
     for (unsigned int thread=0; thread<n_threads; ++thread)
-      threads += Threads::spawn (*this, &Solver<dim>::assemble_matrix)
-                 (linear_system,
-                  thread_ranges[thread].first,
-                  thread_ranges[thread].second,
-                  mutex);
-
-                                    // While the spawned threads
+      threads += Threads::new_thread (&Solver<dim>::assemble_matrix,
+                                     *this,
+                                     linear_system,
+                                     thread_ranges[thread].first,
+                                     thread_ranges[thread].second,
+                                     mutex);
+
+                                    // While the new threads
                                     // assemble the system matrix, we
                                     // can already compute the right
                                     // hand side vector in the main
@@ -1199,8 +1202,8 @@ namespace LaplaceSolver
     assemble_rhs (linear_system.rhs);
     linear_system.hanging_node_constraints.condense (linear_system.rhs);
 
-                                    // And while we're already at it
-                                    // to compute things in parallel,
+                                    // And while we're already
+                                    // computing things in parallel,
                                     // interpolating boundary values
                                     // is one more thing that can be
                                     // done independently, so we do
@@ -1389,7 +1392,7 @@ namespace LaplaceSolver
                                   // concurrency, at least;
                                   // otherwise, the actions are
                                   // performed sequentially). Note
-                                  // that we spawn only one thread,
+                                  // that we start only one thread,
                                   // and do the second action in the
                                   // main thread. Since only one
                                   // thread is generated, we don't
@@ -1446,8 +1449,9 @@ namespace LaplaceSolver
       = &DoFTools::make_hanging_node_constraints;
     
     Threads::Thread<>
-      mhnc_thread = Threads::spawn (mhnc_p)(dof_handler,
-                                            hanging_node_constraints);
+      mhnc_thread = Threads::new_thread (mhnc_p,
+                                        dof_handler,
+                                        hanging_node_constraints);
 
     sparsity_pattern.reinit (dof_handler.n_dofs(),
                             dof_handler.n_dofs(),
index 739242b13eb4027358fa6310462205fc507b50be..54e260fb09176361a05f9338e4c423a58937c76a 100644 (file)
@@ -647,11 +647,12 @@ namespace LaplaceSolver
     Threads::ThreadMutex mutex;
     Threads::ThreadGroup<> threads;
     for (unsigned int thread=0; thread<n_threads; ++thread)
-      threads += Threads::spawn (*this, &Solver<dim>::assemble_matrix)
-                 (linear_system,
-                  thread_ranges[thread].first,
-                  thread_ranges[thread].second,
-                  mutex);
+      threads += Threads::new_thread (&Solver<dim>::assemble_matrix,
+                                     *this,
+                                     linear_system,
+                                     thread_ranges[thread].first,
+                                     thread_ranges[thread].second,
+                                     mutex);
 
     assemble_rhs (linear_system.rhs);
     linear_system.hanging_node_constraints.condense (linear_system.rhs);
@@ -726,7 +727,9 @@ namespace LaplaceSolver
       = &DoFTools::make_hanging_node_constraints;
     
     Threads::Thread<>
-      mhnc_thread = Threads::spawn (mhnc_p)(dof_handler, hanging_node_constraints);
+      mhnc_thread = Threads::new_thread (mhnc_p,
+                                        dof_handler,
+                                        hanging_node_constraints);
 
     sparsity_pattern.reinit (dof_handler.n_dofs(),
                             dof_handler.n_dofs(),
@@ -2397,7 +2400,7 @@ namespace LaplaceSolver
                                       // has a subtle but important
                                       // drawback: we will call these
                                       // functions over and over
-                                      // again, many thousand times
+                                      // again, many thousands of times
                                       // maybe; it has now turned out
                                       // that allocating vectors and
                                       // other objects that need
@@ -2619,8 +2622,10 @@ namespace LaplaceSolver
   WeightedResidual<dim>::solve_problem ()
   {
     Threads::ThreadGroup<> threads;
-    threads += Threads::spawn (*this, &WeightedResidual<dim>::solve_primal_problem)();
-    threads += Threads::spawn (*this, &WeightedResidual<dim>::solve_dual_problem)();
+    threads += Threads::new_thread (&WeightedResidual<dim>::solve_primal_problem,
+                                   *this);
+    threads += Threads::new_thread (&WeightedResidual<dim>::solve_dual_problem,
+                                   *this);
     threads.join_all ();
   }
 
@@ -2989,12 +2994,13 @@ namespace LaplaceSolver
     const unsigned int n_threads = multithread_info.n_default_threads;
     Threads::ThreadGroup<> threads;
     for (unsigned int i=0; i<n_threads; ++i)
-      threads += Threads::spawn (*this, &WeightedResidual<dim>::estimate_some)
-                 (primal_solution,
-                  dual_weights,
-                  n_threads, i,
-                  error_indicators,
-                  face_integrals);
+      threads += Threads::new_thread (&WeightedResidual<dim>::estimate_some,
+                                     *this,
+                                     primal_solution,
+                                     dual_weights,
+                                     n_threads, i,
+                                     error_indicators,
+                                     face_integrals);
     threads.join_all();    
 
                                     // Once the error contributions
index d3fd019d835b82d43357a5c772458ec5e826b44f..123a20dc33def6c38d53d2a3813955789c8f5902 100644 (file)
@@ -1551,10 +1551,12 @@ EnergyGroup<dim>::output_results (const unsigned int cycle) const
                                  // doing all the rest. In several
                                  // places, we have to do something
                                  // for all energy groups, in which
-                                 // case we will spawn threads for
+                                 // case we will start threads for
                                  // each group to let these things run
                                  // in parallel if deal.II was
                                  // configured for multithreading.
+                                // For strategies of parallelization,
+                                // take a look at the @ref threads module.
                                  //
                                  // The biggest difference to previous
                                  // example programs is that we also
@@ -2025,30 +2027,27 @@ void NeutronDiffusionProblem<dim>::initialize_problem()
                                  // iteration. The total power then is
                                  // used to renew k-effective.
                                 //
-                                // Since the total fission source is
-                                // a sum over all the energy groups,
-                                // and since each of these sums can
-                                // be computed independently, we
-                                // actually do this in parallel. One
-                                // of the problems is that the
-                                // function in the
-                                // <code>EnergyGroup</code> class
-                                // that computes the fission source
-                                // returns a value. If we now simply
-                                // spin off a new thread, we have to
-                                // later capture the return value of
-                                // the function run on that
-                                // thread. The way this can be done
-                                // is to use the return value of the
-                                // Threads::spawn function, which is
-                                // of type Threads::Thread@<double@>
-                                // if the function spawned returns a
-                                // double. We can the later ask this
-                                // object for the returned value
+                                // Since the total fission source is a sum
+                                // over all the energy groups, and since each
+                                // of these sums can be computed
+                                // independently, we actually do this in
+                                // parallel. One of the problems is that the
+                                // function in the <code>EnergyGroup</code>
+                                // class that computes the fission source
+                                // returns a value. If we now simply spin off
+                                // a new thread, we have to later capture the
+                                // return value of the function run on that
+                                // thread. The way this can be done is to use
+                                // the return value of the
+                                // Threads::new_thread function, which
+                                // returns an object of type
+                                // Threads::Thread@<double@> if the function
+                                // spawned returns a double. We can then later
+                                // ask this object for the returned value
                                 // (when doing so, the
                                 // Threads::Thread@<double@>::return_value
-                                // function first waits for the
-                                // thread to finish).
+                                // function first waits for the thread to
+                                // finish if it hasn't done so already).
                                 //
                                 // The way this function then works
                                 // is to first spawn one thread for
@@ -2061,8 +2060,8 @@ double NeutronDiffusionProblem<dim>::get_total_fission_source () const
 {
   std::vector<Threads::Thread<double> > threads;
   for (unsigned int group=0; group<parameters.n_groups; ++group)
-    threads.push_back (Threads::spawn (*energy_groups[group],
-                                      &EnergyGroup<dim>::get_fission_source) ());
+    threads.push_back (Threads::new_thread (&EnergyGroup<dim>::get_fission_source,
+                                           *energy_groups[group]));
   
   double fission_source = 0;
   for (unsigned int group=0; group<parameters.n_groups; ++group)
@@ -2103,8 +2102,9 @@ void NeutronDiffusionProblem<dim>::refine_grid ()
   {  
     Threads::ThreadGroup<> threads;
     for (unsigned int group=0; group<parameters.n_groups; ++group)
-      threads += Threads::spawn (*energy_groups[group], &EnergyGroup<dim>::estimate_errors)
-                (group_error_indicators.block(group));
+      threads += Threads::new_thread (&EnergyGroup<dim>::estimate_errors,
+                                     *energy_groups[group],
+                                     group_error_indicators.block(group));
     threads.join_all ();
   }
   
@@ -2115,10 +2115,11 @@ void NeutronDiffusionProblem<dim>::refine_grid ()
   {  
     Threads::ThreadGroup<> threads;
     for (unsigned int group=0; group<parameters.n_groups; ++group)
-      threads += Threads::spawn (*energy_groups[group], &EnergyGroup<dim>::refine_grid)
-                (group_error_indicators.block(group),
-                 refine_threshold,
-                 coarsen_threshold);
+      threads += Threads::new_thread (&EnergyGroup<dim>::refine_grid,
+                                     *energy_groups[group], 
+                                     group_error_indicators.block(group),
+                                     refine_threshold,
+                                     coarsen_threshold);
     threads.join_all ();
   }
 }
@@ -2176,9 +2177,9 @@ void NeutronDiffusionProblem<dim>::run ()
       
       Threads::ThreadGroup<> threads;
       for (unsigned int group=0; group<parameters.n_groups; ++group)
-       threads += Threads::spawn
-                  (*energy_groups[group], &EnergyGroup<dim>::assemble_system_matrix)
-                  ();
+       threads += Threads::new_thread
+                  (&EnergyGroup<dim>::assemble_system_matrix,
+                   *energy_groups[group]);
       threads.join_all ();
 
       double error;
index 593bf5953eb069e789af86edf8090c5f9a58a260..495b6ad68c16ed04cc7dc220d1a5d1772682399e 100644 (file)
@@ -20,6 +20,7 @@
 #include <base/quadrature_lib.h>
 #include <base/logstream.h>
 #include <base/utilities.h>
+#include <base/work_stream.h>
 
 #include <lac/full_matrix.h>
 #include <lac/solver_gmres.h>
@@ -534,6 +535,116 @@ namespace LinearSolvers
 
 
 
+namespace AssemblerData
+{
+  template <int dim>
+  struct StokesPreconditioner 
+  {
+      StokesPreconditioner (const FiniteElement<dim> &stokes_fe,
+                           const Quadrature<dim>    &stokes_quadrature,
+                           const UpdateFlags         update_flags);
+      StokesPreconditioner (const StokesPreconditioner &data);
+       
+      FEValues<dim>               stokes_fe_values;
+
+      FullMatrix<double>          local_matrix;
+      std::vector<unsigned int>   local_dof_indices;
+
+      std::vector<Tensor<2,dim> > grad_phi_u;
+      std::vector<double>         phi_p;
+  };
+
+  template <int dim>
+  StokesPreconditioner<dim>::
+  StokesPreconditioner (const FiniteElement<dim> &stokes_fe,
+                       const Quadrature<dim>    &stokes_quadrature,
+                       const UpdateFlags         update_flags)
+                 :
+                 stokes_fe_values (stokes_fe, stokes_quadrature, update_flags),
+                 local_matrix (stokes_fe.dofs_per_cell, stokes_fe.dofs_per_cell),
+                 local_dof_indices (stokes_fe.dofs_per_cell),
+                 grad_phi_u (stokes_fe.dofs_per_cell),
+                 phi_p (stokes_fe.dofs_per_cell)
+  {}
+
+
+
+  template <int dim>
+  StokesPreconditioner<dim>::
+  StokesPreconditioner (const StokesPreconditioner &data)
+                 :
+                 stokes_fe_values (data.stokes_fe_values.get_fe(),
+                                   data.stokes_fe_values.get_quadrature(),
+                                   data.stokes_fe_values.get_update_flags()),
+                 local_matrix (data.local_matrix),
+                 local_dof_indices (data.local_dof_indices),
+                 grad_phi_u (data.grad_phi_u),
+                 phi_p (data.phi_p)
+  {}
+
+
+
+  template <int dim>
+  struct StokesSystem : public StokesPreconditioner<dim>
+  {
+      StokesSystem (const FiniteElement<dim> &stokes_fe,
+                   const Quadrature<dim>    &stokes_quadrature,
+                   const UpdateFlags         stokes_update_flags,
+                   const FiniteElement<dim> &temperature_fe,
+                   const UpdateFlags         temperature_update_flags);
+
+      StokesSystem (const StokesSystem<dim> &data);
+      
+      FEValues<dim>  temperature_fe_values;
+      Vector<double> local_rhs;
+
+      std::vector<Tensor<1,dim> >          phi_u;
+      std::vector<SymmetricTensor<2,dim> > grads_phi_u;
+      std::vector<double>                  div_phi_u;
+
+      std::vector<double>                  old_temperature_values;
+  };
+  
+
+  template <int dim>
+  StokesSystem<dim>::
+  StokesSystem (const FiniteElement<dim> &stokes_fe,
+               const Quadrature<dim>    &stokes_quadrature,
+               const UpdateFlags         stokes_update_flags,
+               const FiniteElement<dim> &temperature_fe,
+               const UpdateFlags         temperature_update_flags)
+                 :
+                 StokesPreconditioner<dim> (stokes_fe, stokes_quadrature,
+                                            stokes_update_flags),
+                 temperature_fe_values (temperature_fe, stokes_quadrature,
+                                        temperature_update_flags),
+                 local_rhs (stokes_fe.dofs_per_cell),
+                 phi_u (stokes_fe.dofs_per_cell),
+                 grads_phi_u (stokes_fe.dofs_per_cell),
+                 div_phi_u (stokes_fe.dofs_per_cell),
+                 old_temperature_values (stokes_quadrature.n_quadrature_points)
+  {}
+
+
+  template <int dim>
+  StokesSystem<dim>::
+  StokesSystem (const StokesSystem<dim> &data)
+                 :
+                 StokesPreconditioner<dim> (data),
+                 temperature_fe_values (data.temperature_fe_values.get_fe(),
+                                        data.temperature_fe_values.get_quadrature(),
+                                        data.temperature_fe_values.get_update_flags()),
+                 local_rhs (data.local_rhs),
+                 phi_u (data.phi_u),
+                 grads_phi_u (data.grads_phi_u),
+                 div_phi_u (data.div_phi_u),
+                 old_temperature_values (data.old_temperature_values)
+  {}
+}
+
+  
+
+
                                 // @sect3{The <code>BoussinesqFlowProblem</code> class template}
 
                                 // The definition of the class that defines
@@ -607,7 +718,6 @@ class BoussinesqFlowProblem
                      const double                        global_T_variation,
                      const double                        cell_diameter) const;
 
-
     Triangulation<dim>                  triangulation;
     double                              global_Omega_diameter;
 
@@ -650,6 +760,19 @@ class BoussinesqFlowProblem
     bool rebuild_stokes_matrix;
     bool rebuild_temperature_matrices;
     bool rebuild_stokes_preconditioner;
+
+    void
+    local_assemble_stokes_preconditioner (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                         AssemblerData::StokesPreconditioner<dim> &data);
+    
+    void copy_local_to_global_stokes_preconditioner (const AssemblerData::StokesPreconditioner<dim> &data);
+
+
+    void
+    local_assemble_stokes_system (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                 AssemblerData::StokesSystem<dim> &data);
+    
+    void copy_local_to_global_stokes_system (const AssemblerData::StokesSystem<dim> &data);
 };
 
 
@@ -1321,6 +1444,79 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
 
 
 
+
+  
+
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+local_assemble_stokes_preconditioner (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                     AssemblerData::StokesPreconditioner<dim> &data)
+{
+  const unsigned int   dofs_per_cell   = stokes_fe.dofs_per_cell;
+  const unsigned int   n_q_points      = data.stokes_fe_values.n_quadrature_points;
+
+  const FEValuesExtractors::Vector velocities (0);
+  const FEValuesExtractors::Scalar pressure (dim);
+
+  data.stokes_fe_values.reinit (cell);
+  data.local_matrix = 0;
+
+                                  // The creation of the local matrix is
+                                  // rather simple. There are only a
+                                  // Laplace term (on the velocity) and a
+                                  // mass matrix weighted by $\eta^{-1}$
+                                  // to be generated, so the creation of
+                                  // the local matrix is done in two
+                                  // lines. Once the local matrix is
+                                  // ready (loop over rows and columns in
+                                  // the local matrix on each quadrature
+                                  // point), we get the local DoF indices
+                                  // and write the local information into
+                                  // the global matrix. We do this as in
+                                  // step-27, i.e. we directly apply the
+                                  // constraints from hanging nodes
+                                  // locally. By doing so, we don't have
+                                  // to do that afterwards, and we don't
+                                  // also write into entries of the
+                                  // matrix that will actually be set to
+                                  // zero again later when eliminating
+                                  // constraints.
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
+       {
+         data.grad_phi_u[k] = data.stokes_fe_values[velocities].gradient(k,q);
+         data.phi_p[k]      = data.stokes_fe_values[pressure].value (k, q);
+       }
+         
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       for (unsigned int j=0; j<dofs_per_cell; ++j)
+         data.local_matrix(i,j) += (EquationData::eta *
+                                    scalar_product (data.grad_phi_u[i], data.grad_phi_u[j])
+                                    +
+                                    (1./EquationData::eta) *
+                                    data.phi_p[i] * data.phi_p[j])
+                                   * data.stokes_fe_values.JxW(q);
+    }
+
+  cell->get_dof_indices (data.local_dof_indices);
+}
+
+
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_stokes_preconditioner (const AssemblerData::StokesPreconditioner<dim> &data)
+{
+  stokes_constraints.distribute_local_to_global (data.local_matrix,
+                                                data.local_dof_indices,
+                                                stokes_preconditioner_matrix);
+}
+
+
+
+
                                 // @sect4{BoussinesqFlowProblem::assemble_stokes_preconditioner}
                                 // 
                                  // This function assembles the matrix we use
@@ -1336,7 +1532,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
                                  // we create data structures for the cell
                                  // matrix and the relation between local and
                                  // global DoFs. The vectors
-                                 // <code>phi_grad_u</code> and
+                                 // <code>grad_phi_u</code> and
                                  // <code>phi_p</code> are going to hold the
                                  // values of the basis functions in order to
                                  // faster build up the local matrices, as was
@@ -1351,74 +1547,25 @@ BoussinesqFlowProblem<dim>::assemble_stokes_preconditioner ()
   stokes_preconditioner_matrix = 0;
 
   const QGauss<dim> quadrature_formula(stokes_degree+2);
-  FEValues<dim>     stokes_fe_values (stokes_fe, quadrature_formula,
-                                     update_JxW_values |
-                                     update_values |
-                                     update_gradients);
-
-  const unsigned int   dofs_per_cell   = stokes_fe.dofs_per_cell;
-  const unsigned int   n_q_points      = quadrature_formula.size();
-
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
-
-  std::vector<Tensor<2,dim> > phi_grad_u (dofs_per_cell);
-  std::vector<double>         phi_p      (dofs_per_cell);
-
-  const FEValuesExtractors::Vector velocities (0);
-  const FEValuesExtractors::Scalar pressure (dim);
 
-  typename DoFHandler<dim>::active_cell_iterator
-    cell = stokes_dof_handler.begin_active(),
-    endc = stokes_dof_handler.end();
-  for (; cell!=endc; ++cell)
-    {
-      stokes_fe_values.reinit (cell);
-      local_matrix = 0;
-
-                                      // The creation of the local matrix is
-                                      // rather simple. There are only a
-                                      // Laplace term (on the velocity) and a
-                                      // mass matrix weighted by $\eta^{-1}$
-                                      // to be generated, so the creation of
-                                      // the local matrix is done in two
-                                      // lines. Once the local matrix is
-                                      // ready (loop over rows and columns in
-                                      // the local matrix on each quadrature
-                                      // point), we get the local DoF indices
-                                      // and write the local information into
-                                      // the global matrix. We do this as in
-                                      // step-27, i.e. we directly apply the
-                                      // constraints from hanging nodes
-                                      // locally. By doing so, we don't have
-                                      // to do that afterwards, and we don't
-                                      // also write into entries of the
-                                      // matrix that will actually be set to
-                                      // zero again later when eliminating
-                                      // constraints.
-      for (unsigned int q=0; q<n_q_points; ++q)
-       {
-         for (unsigned int k=0; k<dofs_per_cell; ++k)
-           {
-             phi_grad_u[k] = stokes_fe_values[velocities].gradient(k,q);
-             phi_p[k]      = stokes_fe_values[pressure].value (k, q);
-           }
-         
-         for (unsigned int i=0; i<dofs_per_cell; ++i)
-           for (unsigned int j=0; j<dofs_per_cell; ++j)
-             local_matrix(i,j) += (EquationData::eta *
-                                   scalar_product (phi_grad_u[i], phi_grad_u[j])
-                                   +
-                                   (1./EquationData::eta) *
-                                   phi_p[i] * phi_p[j])
-                                  * stokes_fe_values.JxW(q);
-       }
-
-      cell->get_dof_indices (local_dof_indices);
-      stokes_constraints.distribute_local_to_global (local_matrix,
-                                                    local_dof_indices,
-                                                    stokes_preconditioner_matrix);
-    }
+  AssemblerData::StokesPreconditioner<dim>
+    data_template (stokes_fe, quadrature_formula,
+                  update_JxW_values |
+                  update_values |
+                  update_gradients);
+
+  WorkStream().run (stokes_dof_handler.begin_active(),
+                   stokes_dof_handler.end(),
+                   std_cxx0x::bind (&BoussinesqFlowProblem<dim>::
+                                    local_assemble_stokes_preconditioner,
+                                    this,
+                                    _1,
+                                    _2),
+                   std_cxx0x::bind (&BoussinesqFlowProblem<dim>::
+                                    copy_local_to_global_stokes_preconditioner,
+                                    this,
+                                    _1),
+                   data_template);  
 }
 
 
@@ -1623,37 +1770,14 @@ BoussinesqFlowProblem<dim>::build_stokes_preconditioner ()
                                 // the local dofs compared to the global
                                 // system.
 template <int dim>
-void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
+void
+BoussinesqFlowProblem<dim>::
+local_assemble_stokes_system (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                             AssemblerData::StokesSystem<dim> &data)
 {
-  std::cout << "   Assembling..." << std::flush;
-
-  if (rebuild_stokes_matrix == true)
-    stokes_matrix=0;
-
-  stokes_rhs=0;
-
-  const QGauss<dim> quadrature_formula (stokes_degree+2);
-  FEValues<dim>     stokes_fe_values (stokes_fe, quadrature_formula,
-                                     update_values    |
-                                     update_quadrature_points  |
-                                     update_JxW_values |
-                                     (rebuild_stokes_matrix == true
-                                      ?
-                                      update_gradients
-                                      :
-                                      UpdateFlags(0)));
+  const unsigned int   dofs_per_cell   = data.stokes_fe_values.get_fe().dofs_per_cell;
+  const unsigned int   n_q_points      = data.stokes_fe_values.n_quadrature_points;
   
-  FEValues<dim>     temperature_fe_values (temperature_fe, quadrature_formula,
-                                          update_values);
-
-  const unsigned int   dofs_per_cell   = stokes_fe.dofs_per_cell;
-  const unsigned int   n_q_points      = quadrature_formula.size();
-
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
-  Vector<double>       local_rhs    (dofs_per_cell);
-
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
-
                                   // Next we need a vector that will contain
                                   // the values of the temperature solution
                                   // at the previous time level at the
@@ -1677,12 +1801,6 @@ void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
                                   // extract the individual blocks
                                   // (velocity, pressure, temperature) from
                                   // the total FE system.
-  std::vector<double>               old_temperature_values(n_q_points);
-
-  std::vector<Tensor<1,dim> >          phi_u       (dofs_per_cell);
-  std::vector<SymmetricTensor<2,dim> > grads_phi_u (dofs_per_cell);
-  std::vector<double>                  div_phi_u   (dofs_per_cell);
-  std::vector<double>                  phi_p       (dofs_per_cell);
 
   const FEValuesExtractors::Vector velocities (0);
   const FEValuesExtractors::Scalar pressure (dim);
@@ -1706,103 +1824,147 @@ void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
                                   // quadrature points. Then we are ready to
                                   // loop over the quadrature points on the
                                   // cell.
+  data.stokes_fe_values.reinit (cell);
+
   typename DoFHandler<dim>::active_cell_iterator
-    cell = stokes_dof_handler.begin_active(),
-    endc = stokes_dof_handler.end();
-  typename DoFHandler<dim>::active_cell_iterator
-    temperature_cell = temperature_dof_handler.begin_active();
-  
-  for (; cell!=endc; ++cell, ++temperature_cell)
-    {
-      stokes_fe_values.reinit (cell);
-      temperature_fe_values.reinit (temperature_cell);
+    temperature_cell (&triangulation,
+                     cell->level(),
+                     cell->index(),
+                     &temperature_dof_handler);
+  data.temperature_fe_values.reinit (temperature_cell);
       
-      local_matrix = 0;
-      local_rhs = 0;
+  data.local_matrix = 0;
+  data.local_rhs = 0;
 
-      temperature_fe_values.get_function_values (old_temperature_solution, 
-                                                old_temperature_values);
+  data.temperature_fe_values.get_function_values (old_temperature_solution, 
+                                                 data.old_temperature_values);
 
-      for (unsigned int q=0; q<n_q_points; ++q)
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      const double old_temperature = data.old_temperature_values[q];
+
+                                      // Next we extract the values and
+                                      // gradients of basis functions
+                                      // relevant to the terms in the
+                                      // inner products. As shown in
+                                      // step-22 this helps accelerate
+                                      // assembly.
+                                      // 
+                                      // Once this is done, we start the
+                                      // loop over the rows and columns
+                                      // of the local matrix and feed the
+                                      // matrix with the relevant
+                                      // products. The right hand side is
+                                      // filled with the forcing term
+                                      // driven by temperature in
+                                      // direction of gravity (which is
+                                      // vertical in our example).  Note
+                                      // that the right hand side term is
+                                      // always generated, whereas the
+                                      // matrix contributions are only
+                                      // updated when it is requested by
+                                      // the
+                                      // <code>rebuild_matrices</code>
+                                      // flag.
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
        {
-         const double old_temperature = old_temperature_values[q];
-
-                                          // Next we extract the values and
-                                          // gradients of basis functions
-                                          // relevant to the terms in the
-                                          // inner products. As shown in
-                                          // step-22 this helps accelerate
-                                          // assembly.
-                                          // 
-                                          // Once this is done, we start the
-                                          // loop over the rows and columns
-                                          // of the local matrix and feed the
-                                          // matrix with the relevant
-                                          // products. The right hand side is
-                                          // filled with the forcing term
-                                          // driven by temperature in
-                                          // direction of gravity (which is
-                                          // vertical in our example).  Note
-                                          // that the right hand side term is
-                                          // always generated, whereas the
-                                          // matrix contributions are only
-                                          // updated when it is requested by
-                                          // the
-                                          // <code>rebuild_matrices</code>
-                                          // flag.
-         for (unsigned int k=0; k<dofs_per_cell; ++k)
+         data.phi_u[k] = data.stokes_fe_values[velocities].value (k,q);
+         if (rebuild_stokes_matrix)
            {
-             phi_u[k] = stokes_fe_values[velocities].value (k,q);
-             if (rebuild_stokes_matrix)
-               {
-                 grads_phi_u[k] = stokes_fe_values[velocities].symmetric_gradient(k,q);
-                 div_phi_u[k]   = stokes_fe_values[velocities].divergence (k, q);
-                 phi_p[k]       = stokes_fe_values[pressure].value (k, q);
-               }
+             data.grads_phi_u[k] = data.stokes_fe_values[velocities].symmetric_gradient(k,q);
+             data.div_phi_u[k]   = data.stokes_fe_values[velocities].divergence (k, q);
+             data.phi_p[k]       = data.stokes_fe_values[pressure].value (k, q);
            }
-
-         if (rebuild_stokes_matrix)
-           for (unsigned int i=0; i<dofs_per_cell; ++i)
-             for (unsigned int j=0; j<dofs_per_cell; ++j)
-               local_matrix(i,j) += (EquationData::eta * 2 *
-                                     grads_phi_u[i] * grads_phi_u[j]
-                                     - div_phi_u[i] * phi_p[j]
-                                     - phi_p[i] * div_phi_u[j])
-                                    * stokes_fe_values.JxW(q);
-
-         const Point<dim> gravity = ( (dim == 2) ? (Point<dim> (0,1)) : 
-                                      (Point<dim> (0,0,1)) );
-         for (unsigned int i=0; i<dofs_per_cell; ++i)
-           local_rhs(i) += (EquationData::Rayleigh_number *
-                            gravity * phi_u[i] * old_temperature)*
-                           stokes_fe_values.JxW(q);
        }
 
-                                      // The last step in the loop over all
-                                      // cells is to enter the local
-                                      // contributions into the global matrix
-                                      // and vector structures to the
-                                      // positions specified in
-                                      // <code>local_dof_indices</code>.
-                                      // Again, we let the ConstraintMatrix
-                                      // class do the insertion of the cell
-                                      // matrix elements to the global
-                                      // matrix, which already condenses the
-                                      // hanging node constraints.
-      cell->get_dof_indices (local_dof_indices);
-
-      if (rebuild_stokes_matrix == true)
-       stokes_constraints.distribute_local_to_global (local_matrix,
-                                                      local_rhs,
-                                                      local_dof_indices,
-                                                      stokes_matrix,
-                                                      stokes_rhs);
-      else
-       stokes_constraints.distribute_local_to_global (local_rhs,
-                                                      local_dof_indices,
-                                                      stokes_rhs);
+      if (rebuild_stokes_matrix)
+       for (unsigned int i=0; i<dofs_per_cell; ++i)
+         for (unsigned int j=0; j<dofs_per_cell; ++j)
+           data.local_matrix(i,j) += (EquationData::eta * 2 *
+                                      data.grads_phi_u[i] * data.grads_phi_u[j]
+                                      - data.div_phi_u[i] * data.phi_p[j]
+                                      - data.phi_p[i] * data.div_phi_u[j])
+                                     * data.stokes_fe_values.JxW(q);
+
+      const Point<dim> gravity = ( (dim == 2) ? (Point<dim> (0,1)) : 
+                                  (Point<dim> (0,0,1)) );
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       data.local_rhs(i) += (EquationData::Rayleigh_number *
+                             gravity * data.phi_u[i] * old_temperature)*
+                            data.stokes_fe_values.JxW(q);
     }
 
+                                  // The last step in the loop over all
+                                  // cells is to enter the local
+                                  // contributions into the global matrix
+                                  // and vector structures to the
+                                  // positions specified in
+                                  // <code>local_dof_indices</code>.
+                                  // Again, we let the ConstraintMatrix
+                                  // class do the insertion of the cell
+                                  // matrix elements to the global
+                                  // matrix, which already condenses the
+                                  // hanging node constraints.
+  cell->get_dof_indices (data.local_dof_indices);
+}
+
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_stokes_system (const AssemblerData::StokesSystem<dim> &data)
+{
+  if (rebuild_stokes_matrix == true)
+    stokes_constraints.distribute_local_to_global (local_matrix,
+                                                  local_rhs,
+                                                  local_dof_indices,
+                                                  stokes_matrix,
+                                                  stokes_rhs);
+  else
+    stokes_constraints.distribute_local_to_global (local_rhs,
+                                                  local_dof_indices,
+                                                  stokes_rhs);
+}
+
+
+
+template <int dim>
+void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
+{
+  std::cout << "   Assembling..." << std::flush;
+
+  if (rebuild_stokes_matrix == true)
+    stokes_matrix=0;
+
+  stokes_rhs=0;
+
+  const QGauss<dim> quadrature_formula (stokes_degree+2);
+
+  AssemblerData::StokesSystem<dim>
+    data_template (stokes_fe, quadrature_formula,
+                  (update_values    |
+                   update_quadrature_points  |
+                   update_JxW_values |
+                   (rebuild_stokes_matrix == true
+                    ?
+                    update_gradients
+                    :
+                    UpdateFlags(0))),
+                  temperature_fe,
+                  update_values);
+
+  WorkStream().run (stokes_dof_handler.begin_active(),
+                   stokes_dof_handler.end(),
+                   std_cxx0x::bind (&BoussinesqFlowProblem<dim>::
+                                    local_assemble_stokes_system,
+                                    this,
+                                    _1,
+                                    _2),
+                   std_cxx0x::bind (&BoussinesqFlowProblem<dim>::
+                                    copy_local_to_global_stokes_system,
+                                    this,
+                                    _1),
+                   data_template);  
+
   rebuild_stokes_matrix = false;
 
   std::cout << std::endl;
@@ -1994,7 +2156,6 @@ void BoussinesqFlowProblem<dim>::
   const unsigned int   n_q_points      = quadrature_formula.size();
 
   Vector<double>       local_rhs (dofs_per_cell);
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
 
   std::vector<unsigned int> local_dof_indices (dofs_per_cell);
 
index 4b0b291cfa2fbdf8ea6ab5e524e1d9f63753a3ab..c92672ba87b8e2cf505db239eee2c273202147f2 100644 (file)
 #include <base/function.h>
 #include <base/utilities.h>
 #include <base/conditional_ostream.h>
+#include <base/work_stream.h>
 
 #include <lac/full_matrix.h>
-#include <lac/solver_gmres.h>
+#include <lac/solver_bicgstab.h>
 #include <lac/solver_cg.h>
 #include <lac/constraint_matrix.h>
 #include <lac/block_sparsity_pattern.h>
@@ -35,6 +36,7 @@
 #include <grid/grid_generator.h>
 #include <grid/tria_accessor.h>
 #include <grid/tria_iterator.h>
+#include <grid/filtered_iterator.h>
 #include <grid/tria_boundary_lib.h>
 #include <grid/grid_tools.h>
 #include <grid/grid_refinement.h>
@@ -57,7 +59,7 @@
 
 #include <Epetra_Map.h>
 
-                                       // Time measurements. 
+                                // Time measurements. 
 #include <base/timer.h>
 
 #include <fstream>
@@ -196,7 +198,7 @@ namespace LinearSolvers
       const PreconditionerMp &mp_preconditioner;
       const PreconditionerA  &a_preconditioner;
       mutable TrilinosWrappers::MPI::Vector tmp;
-};
+  };
 
 
 
@@ -206,10 +208,10 @@ namespace LinearSolvers
                           const PreconditionerMp                    &Mppreconditioner,
                           const PreconditionerA                     &Apreconditioner)
                  :
-                 stokes_matrix           (&S),
-                 mp_preconditioner       (Mppreconditioner),
-                 a_preconditioner        (Apreconditioner),
-                 tmp                     (stokes_matrix->block(1,1).row_partitioner())
+                 stokes_matrix     (&S),
+                 mp_preconditioner (Mppreconditioner),
+                 a_preconditioner  (Apreconditioner),
+                 tmp               (stokes_matrix->block(1,1).row_partitioner())
   {}
 
 
@@ -228,6 +230,356 @@ namespace LinearSolvers
 
 
 
+namespace Assembly
+{
+  namespace Scratch
+  {
+    template <int dim>
+    struct StokesPreconditioner 
+    {
+       StokesPreconditioner (const FiniteElement<dim> &stokes_fe,
+                             const Quadrature<dim>    &stokes_quadrature,
+                             const UpdateFlags         update_flags);
+       StokesPreconditioner (const StokesPreconditioner &data);
+       
+       FEValues<dim>               stokes_fe_values;
+
+       std::vector<Tensor<2,dim> > grad_phi_u;
+       std::vector<double>         phi_p;
+    };
+
+    template <int dim>
+    StokesPreconditioner<dim>::
+    StokesPreconditioner (const FiniteElement<dim> &stokes_fe,
+                         const Quadrature<dim>    &stokes_quadrature,
+                         const UpdateFlags         update_flags)
+                   :
+                   stokes_fe_values (stokes_fe, stokes_quadrature,
+                                     update_flags),
+                   grad_phi_u (stokes_fe.dofs_per_cell),
+                   phi_p (stokes_fe.dofs_per_cell)
+    {}
+
+
+
+    template <int dim>
+    StokesPreconditioner<dim>::
+    StokesPreconditioner (const StokesPreconditioner &scratch)
+                   :
+                   stokes_fe_values (scratch.stokes_fe_values.get_fe(),
+                                     scratch.stokes_fe_values.get_quadrature(),
+                                     scratch.stokes_fe_values.get_update_flags()),
+                   grad_phi_u (scratch.grad_phi_u),
+                   phi_p (scratch.phi_p)
+    {}
+
+
+
+    template <int dim>
+    struct StokesSystem : public StokesPreconditioner<dim>
+    {
+       StokesSystem (const FiniteElement<dim> &stokes_fe,
+                     const Quadrature<dim>    &stokes_quadrature,
+                     const UpdateFlags         stokes_update_flags,
+                     const FiniteElement<dim> &temperature_fe,
+                     const UpdateFlags         temperature_update_flags);
+
+       StokesSystem (const StokesSystem<dim> &data);
+      
+       FEValues<dim>  temperature_fe_values;
+
+       std::vector<Tensor<1,dim> >          phi_u;
+       std::vector<SymmetricTensor<2,dim> > grads_phi_u;
+       std::vector<double>                  div_phi_u;
+
+       std::vector<double>                  old_temperature_values;
+    };
+  
+
+    template <int dim>
+    StokesSystem<dim>::
+    StokesSystem (const FiniteElement<dim> &stokes_fe,
+                 const Quadrature<dim>    &stokes_quadrature,
+                 const UpdateFlags         stokes_update_flags,
+                 const FiniteElement<dim> &temperature_fe,
+                 const UpdateFlags         temperature_update_flags)
+                   :
+                   StokesPreconditioner<dim> (stokes_fe, stokes_quadrature,
+                                              stokes_update_flags),
+                   temperature_fe_values (temperature_fe, stokes_quadrature,
+                                          temperature_update_flags),
+                   phi_u (stokes_fe.dofs_per_cell),
+                   grads_phi_u (stokes_fe.dofs_per_cell),
+                   div_phi_u (stokes_fe.dofs_per_cell),
+                   old_temperature_values (stokes_quadrature.n_quadrature_points)
+    {}
+
+
+    template <int dim>
+    StokesSystem<dim>::
+    StokesSystem (const StokesSystem<dim> &scratch)
+                   :
+                   StokesPreconditioner<dim> (scratch),
+                   temperature_fe_values (scratch.temperature_fe_values.get_fe(),
+                                          scratch.temperature_fe_values.get_quadrature(),
+                                          scratch.temperature_fe_values.get_update_flags()),
+                   phi_u (scratch.phi_u),
+                   grads_phi_u (scratch.grads_phi_u),
+                   div_phi_u (scratch.div_phi_u),
+                   old_temperature_values (scratch.old_temperature_values)
+    {}
+
+
+
+    template <int dim>
+    struct TemperatureMatrix 
+    {
+       TemperatureMatrix (const FiniteElement<dim> &temperature_fe,
+                          const Quadrature<dim>    &temperature_quadrature);
+       TemperatureMatrix (const TemperatureMatrix &data);
+       
+       FEValues<dim>               temperature_fe_values;
+
+       std::vector<double>         phi_T;
+       std::vector<Tensor<1,dim> > grad_phi_T;
+    };
+
+    template <int dim>
+    TemperatureMatrix<dim>::
+    TemperatureMatrix (const FiniteElement<dim> &temperature_fe,
+                      const Quadrature<dim>    &temperature_quadrature)
+                   :
+                   temperature_fe_values (temperature_fe, temperature_quadrature,
+                                          update_values    | update_gradients |
+                                          update_JxW_values),
+                   phi_T (temperature_fe.dofs_per_cell),
+                   grad_phi_T (temperature_fe.dofs_per_cell)
+    {}
+
+
+    template <int dim>
+    TemperatureMatrix<dim>::
+    TemperatureMatrix (const TemperatureMatrix &scratch)
+                   :
+                   temperature_fe_values (scratch.temperature_fe_values.get_fe(),
+                                          scratch.temperature_fe_values.get_quadrature(),
+                                          scratch.temperature_fe_values.get_update_flags()),
+                   phi_T (scratch.phi_T),
+                   grad_phi_T (scratch.grad_phi_T)
+    {}
+
+
+    template <int dim>
+    struct TemperatureRHS 
+    {
+       TemperatureRHS (const FiniteElement<dim> &temperature_fe,
+                       const FiniteElement<dim> &stokes_fe,
+                       const Quadrature<dim>    &quadrature);
+       TemperatureRHS (const TemperatureRHS &data);
+       
+       FEValues<dim>               temperature_fe_values;
+       FEValues<dim>               stokes_fe_values;
+
+       std::vector<double>         phi_T;
+       std::vector<Tensor<1,dim> > grad_phi_T;
+
+       std::vector<Tensor<1,dim> > old_velocity_values;
+       std::vector<Tensor<1,dim> > old_old_velocity_values;
+  
+       std::vector<double>         old_temperature_values;
+       std::vector<double>         old_old_temperature_values;
+       std::vector<Tensor<1,dim> > old_temperature_grads;
+       std::vector<Tensor<1,dim> > old_old_temperature_grads;
+       std::vector<double>         old_temperature_laplacians;
+       std::vector<double>         old_old_temperature_laplacians;
+
+       std::vector<double>         gamma_values;
+    };
+
+    template <int dim>
+    TemperatureRHS<dim>::
+    TemperatureRHS (const FiniteElement<dim> &temperature_fe,
+                   const FiniteElement<dim> &stokes_fe,
+                   const Quadrature<dim>    &quadrature)
+                   :
+                   temperature_fe_values (temperature_fe, quadrature,
+                                          update_values    |
+                                          update_gradients |
+                                          update_hessians  |
+                                          update_quadrature_points |
+                                          update_JxW_values),
+                   stokes_fe_values (stokes_fe, quadrature,
+                                     update_values),
+                   phi_T (temperature_fe.dofs_per_cell),
+                   grad_phi_T (temperature_fe.dofs_per_cell),
+               
+                   old_velocity_values (quadrature.n_quadrature_points),
+                   old_old_velocity_values (quadrature.n_quadrature_points),
+               
+                   old_temperature_values (quadrature.n_quadrature_points),
+                   old_old_temperature_values(quadrature.n_quadrature_points),
+                   old_temperature_grads(quadrature.n_quadrature_points),
+                   old_old_temperature_grads(quadrature.n_quadrature_points),
+                   old_temperature_laplacians(quadrature.n_quadrature_points),
+                   old_old_temperature_laplacians(quadrature.n_quadrature_points),
+               
+                   gamma_values (quadrature.n_quadrature_points)               
+    {}
+
+
+    template <int dim>
+    TemperatureRHS<dim>::
+    TemperatureRHS (const TemperatureRHS &scratch)
+                   :
+                   temperature_fe_values (scratch.temperature_fe_values.get_fe(),
+                                          scratch.temperature_fe_values.get_quadrature(),
+                                          scratch.temperature_fe_values.get_update_flags()),
+                   stokes_fe_values (scratch.stokes_fe_values.get_fe(),
+                                     scratch.stokes_fe_values.get_quadrature(),
+                                     scratch.stokes_fe_values.get_update_flags()),
+                   phi_T (scratch.phi_T),
+                   grad_phi_T (scratch.grad_phi_T),
+               
+                   old_velocity_values (scratch.old_velocity_values),
+                   old_old_velocity_values (scratch.old_old_velocity_values),
+               
+                   old_temperature_values (scratch.old_temperature_values),
+                   old_old_temperature_values (scratch.old_old_temperature_values),
+                   old_temperature_grads (scratch.old_temperature_grads),
+                   old_old_temperature_grads (scratch.old_old_temperature_grads),
+                   old_temperature_laplacians (scratch.old_temperature_laplacians),
+                   old_old_temperature_laplacians (scratch.old_old_temperature_laplacians),
+               
+                   gamma_values (scratch.gamma_values)
+    {}
+  }
+
+  namespace CopyData
+  {
+    template <int dim>
+    struct StokesPreconditioner 
+    {
+       StokesPreconditioner (const FiniteElement<dim> &stokes_fe);
+       StokesPreconditioner (const StokesPreconditioner &data);
+
+       FullMatrix<double>          local_matrix;
+       std::vector<unsigned int>   local_dof_indices;
+    };
+
+    template <int dim>
+    StokesPreconditioner<dim>::
+    StokesPreconditioner (const FiniteElement<dim> &stokes_fe)
+                   :
+                   local_matrix (stokes_fe.dofs_per_cell, 
+                                 stokes_fe.dofs_per_cell),
+                   local_dof_indices (stokes_fe.dofs_per_cell)
+    {}
+
+
+
+    template <int dim>
+    StokesPreconditioner<dim>::
+    StokesPreconditioner (const StokesPreconditioner &data)
+                   :
+                   local_matrix (data.local_matrix),
+                   local_dof_indices (data.local_dof_indices)
+    {}
+
+
+
+    template <int dim>
+    struct StokesSystem : public StokesPreconditioner<dim>
+    {
+       StokesSystem (const FiniteElement<dim> &stokes_fe);
+       StokesSystem (const StokesSystem<dim> &data);
+      
+       Vector<double> local_rhs;
+    };
+  
+
+    template <int dim>
+    StokesSystem<dim>::
+    StokesSystem (const FiniteElement<dim> &stokes_fe)
+                   :
+                   StokesPreconditioner<dim> (stokes_fe),
+                   local_rhs (stokes_fe.dofs_per_cell)
+    {}
+
+
+    template <int dim>
+    StokesSystem<dim>::
+    StokesSystem (const StokesSystem<dim> &data)
+                   :
+                   StokesPreconditioner<dim> (data),
+                   local_rhs (data.local_rhs)
+    {}
+
+
+
+    template <int dim>
+    struct TemperatureMatrix 
+    {
+       TemperatureMatrix (const FiniteElement<dim> &temperature_fe);
+       TemperatureMatrix (const TemperatureMatrix &data);
+       
+       FullMatrix<double>          local_mass_matrix;
+       FullMatrix<double>          local_stiffness_matrix;
+       std::vector<unsigned int>   local_dof_indices;
+    };
+
+    template <int dim>
+    TemperatureMatrix<dim>::
+    TemperatureMatrix (const FiniteElement<dim> &temperature_fe)
+                   :
+                   local_mass_matrix (temperature_fe.dofs_per_cell,
+                                      temperature_fe.dofs_per_cell),
+                   local_stiffness_matrix (temperature_fe.dofs_per_cell,
+                                           temperature_fe.dofs_per_cell),
+                   local_dof_indices (temperature_fe.dofs_per_cell)
+    {}
+
+
+    template <int dim>
+    TemperatureMatrix<dim>::
+    TemperatureMatrix (const TemperatureMatrix &data)
+                   :
+                   local_mass_matrix (data.local_mass_matrix),
+                   local_stiffness_matrix (data.local_stiffness_matrix),
+                   local_dof_indices (data.local_dof_indices)
+    {}
+
+
+    template <int dim>
+    struct TemperatureRHS 
+    {
+       TemperatureRHS (const FiniteElement<dim> &temperature_fe);
+       TemperatureRHS (const TemperatureRHS &data);
+       
+       Vector<double>              local_rhs;
+       std::vector<unsigned int>   local_dof_indices;
+    };
+
+    template <int dim>
+    TemperatureRHS<dim>::
+    TemperatureRHS (const FiniteElement<dim> &temperature_fe)
+                   :
+                   local_rhs (temperature_fe.dofs_per_cell),
+                   local_dof_indices (temperature_fe.dofs_per_cell)
+    {}
+
+
+    template <int dim>
+    TemperatureRHS<dim>::
+    TemperatureRHS (const TemperatureRHS &data)
+                   :
+                   local_rhs (data.local_rhs),
+                   local_dof_indices (data.local_dof_indices)
+    {}
+  }  
+}
+
+
+
                                 // @sect3{The <code>BoussinesqFlowProblem</code> class template}
 template <int dim>
 class BoussinesqFlowProblem
@@ -281,7 +633,7 @@ class BoussinesqFlowProblem
     TrilinosWrappers::BlockSparseMatrix stokes_matrix;
     TrilinosWrappers::BlockSparseMatrix stokes_preconditioner_matrix;
 
-    TrilinosWrappers::MPI::BlockVector  stokes_solution;
+    TrilinosWrappers::BlockVector       stokes_solution;
     TrilinosWrappers::BlockVector       old_stokes_solution;
     TrilinosWrappers::MPI::BlockVector  stokes_rhs;
 
@@ -296,7 +648,7 @@ class BoussinesqFlowProblem
     TrilinosWrappers::SparseMatrix      temperature_stiffness_matrix;
     TrilinosWrappers::SparseMatrix      temperature_matrix;
 
-    TrilinosWrappers::MPI::Vector       temperature_solution;
+    TrilinosWrappers::Vector            temperature_solution;
     TrilinosWrappers::Vector            old_temperature_solution;
     TrilinosWrappers::Vector            old_old_temperature_solution;
     TrilinosWrappers::MPI::Vector       temperature_rhs;
@@ -307,8 +659,8 @@ class BoussinesqFlowProblem
     unsigned int timestep_number;
 
     std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionAMG> Amg_preconditioner;
-    std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU> Mp_preconditioner;
-    std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU> T_preconditioner;
+    std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU>  Mp_preconditioner;
+    std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU>  T_preconditioner;
 
     bool rebuild_stokes_matrix;
     bool rebuild_stokes_preconditioner;
@@ -316,6 +668,44 @@ class BoussinesqFlowProblem
     bool rebuild_temperature_preconditioner;
 
     TimerOutput computing_timer;
+
+    void setup_stokes_matrix ();
+    void setup_stokes_preconditioner ();
+    void setup_temperature_matrices ();
+    
+    void
+    local_assemble_stokes_preconditioner (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                         Assembly::Scratch::StokesPreconditioner<dim> &scratch,
+                                         Assembly::CopyData::StokesPreconditioner<dim> &data);
+    
+    void copy_local_to_global_stokes_preconditioner (const Assembly::CopyData::StokesPreconditioner<dim> &data);
+
+
+    void
+    local_assemble_stokes_system (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                 Assembly::Scratch::StokesSystem<dim>  &scratch,
+                                 Assembly::CopyData::StokesSystem<dim> &data);
+    
+    void copy_local_to_global_stokes_system (const Assembly::CopyData::StokesSystem<dim> &data);
+
+
+    void
+    local_assemble_temperature_matrix (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                      Assembly::Scratch::TemperatureMatrix<dim>  &scratch,
+                                      Assembly::CopyData::TemperatureMatrix<dim> &data);
+    
+    void copy_local_to_global_temperature_matrix (const Assembly::CopyData::TemperatureMatrix<dim> &data);
+
+
+
+    void
+    local_assemble_temperature_rhs (const std::pair<double,double> global_T_range,
+                                   const double                   global_max_velocity,
+                                   const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                   Assembly::Scratch::TemperatureRHS<dim> &scratch,
+                                   Assembly::CopyData::TemperatureRHS<dim> &data);
+    
+    void copy_local_to_global_temperature_rhs (const Assembly::CopyData::TemperatureRHS<dim> &data);
 };
 
 
@@ -326,8 +716,7 @@ template <int dim>
 BoussinesqFlowProblem<dim>::BoussinesqFlowProblem ()
                 :
                 trilinos_communicator (Utilities::Trilinos::comm_world()),
-               pcout (std::cout,
-                      Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)==0),
+               pcout (std::cout, Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)==0),
 
                triangulation (Triangulation<dim>::maximum_smoothing),
 
@@ -363,8 +752,6 @@ double BoussinesqFlowProblem<dim>::get_maximal_velocity () const
                                           stokes_degree+1);
   const unsigned int n_q_points = quadrature_formula.size();
 
-  BlockVector<double> localized_stokes_solution (stokes_solution);
-
   FEValues<dim> fe_values (stokes_fe, quadrature_formula, update_values);
   std::vector<Tensor<1,dim> > velocity_values(n_q_points);
 
@@ -380,7 +767,7 @@ double BoussinesqFlowProblem<dim>::get_maximal_velocity () const
        Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
       {
        fe_values.reinit (cell);
-       fe_values[velocities].get_function_values (localized_stokes_solution,
+       fe_values[velocities].get_function_values (stokes_solution,
                                                   velocity_values);
 
        for (unsigned int q=0; q<n_q_points; ++q)
@@ -551,6 +938,89 @@ compute_viscosity (const std::vector<double>          &old_temperature,
 }
 
 
+template <int dim>
+void BoussinesqFlowProblem<dim>::setup_stokes_matrix ()
+{
+  stokes_matrix.clear ();
+
+  TrilinosWrappers::BlockSparsityPattern sp (stokes_partitioner);
+
+  Table<2,DoFTools::Coupling> coupling (dim+1, dim+1);
+
+  for (unsigned int c=0; c<dim+1; ++c)
+    for (unsigned int d=0; d<dim+1; ++d)
+      if (! ((c==dim) && (d==dim)))
+       coupling[c][d] = DoFTools::always;
+      else
+       coupling[c][d] = DoFTools::none;
+
+  DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp,
+                                  stokes_constraints, false,
+                                  Utilities::Trilinos::
+                                  get_this_mpi_process(trilinos_communicator));
+  sp.compress();
+
+  stokes_matrix.reinit (sp);
+                                  /*std::cout << "Processor " << trilinos_communicator.MyPID() 
+                                    << " stokes(0,0) rows: " 
+                                    << stokes_matrix.block(0,0).matrix->NumMyRows()
+                                    << ", nnz: " 
+                                    << stokes_matrix.block(0,0).matrix->NumMyNonzeros() 
+                                    << std::endl;*/
+
+}
+
+
+
+template <int dim>
+void BoussinesqFlowProblem<dim>::setup_stokes_preconditioner ()
+{
+  Amg_preconditioner.reset ();
+  Mp_preconditioner.reset ();
+    
+  stokes_preconditioner_matrix.clear ();
+
+  TrilinosWrappers::BlockSparsityPattern sp (stokes_partitioner);
+
+  Table<2,DoFTools::Coupling> coupling (dim+1, dim+1);
+  for (unsigned int c=0; c<dim+1; ++c)
+    for (unsigned int d=0; d<dim+1; ++d)
+      if (c == d)
+       coupling[c][d] = DoFTools::always;
+      else
+       coupling[c][d] = DoFTools::none;
+
+  DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp,
+                                  stokes_constraints, false,
+                                  Utilities::Trilinos::
+                                  get_this_mpi_process(trilinos_communicator));
+  sp.compress();
+
+  stokes_preconditioner_matrix.reinit (sp);
+}
+
+
+template <int dim>
+void BoussinesqFlowProblem<dim>::setup_temperature_matrices ()
+{
+  T_preconditioner.reset ();
+  temperature_mass_matrix.clear ();
+  temperature_stiffness_matrix.clear ();
+  temperature_matrix.clear ();
+
+  TrilinosWrappers::SparsityPattern sp (temperature_partitioner);
+  DoFTools::make_sparsity_pattern (temperature_dof_handler, sp,
+                                  temperature_constraints, false,
+                                  Utilities::Trilinos::
+                                  get_this_mpi_process(trilinos_communicator));
+  sp.compress();
+
+  temperature_matrix.reinit (sp);
+  temperature_mass_matrix.reinit (sp);
+  temperature_stiffness_matrix.reinit (sp);
+}
+
+
 
 
                                 // @sect4{BoussinesqFlowProblem::setup_dofs}
@@ -561,7 +1031,8 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
   std::vector<unsigned int> stokes_sub_blocks (dim+1,0);
   stokes_sub_blocks[dim] = 1;
 
-  GridTools::partition_triangulation (Utilities::Trilinos::get_n_mpi_processes(trilinos_communicator), 
+  GridTools::partition_triangulation (Utilities::Trilinos::
+                                     get_n_mpi_processes(trilinos_communicator), 
                                      triangulation);
 
   {
@@ -588,7 +1059,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
                                             temperature_constraints);
     temperature_constraints.close ();
   }
-  
+
   std::vector<unsigned int> stokes_dofs_per_block (2);
   DoFTools::count_dofs_per_block (stokes_dof_handler, stokes_dofs_per_block,
                                  stokes_sub_blocks);
@@ -628,53 +1099,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
     Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_communicator);
     stokes_partitioner.push_back (map_p);
   }
-  {
-    stokes_matrix.clear ();
-
-    TrilinosWrappers::BlockSparsityPattern sp (stokes_partitioner);
-
-    Table<2,DoFTools::Coupling> coupling (dim+1, dim+1);
-
-    for (unsigned int c=0; c<dim+1; ++c)
-      for (unsigned int d=0; d<dim+1; ++d)
-       if (! ((c==dim) && (d==dim)))
-         coupling[c][d] = DoFTools::always;
-       else
-         coupling[c][d] = DoFTools::none;
-
-    DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp,
-                                    stokes_constraints, false,
-                                    Utilities::Trilinos::
-                                    get_this_mpi_process(trilinos_communicator));
-    sp.compress();
-
-    stokes_matrix.reinit (sp);
-  }
-
-  {
-    Amg_preconditioner.reset ();
-    Mp_preconditioner.reset ();
-    stokes_preconditioner_matrix.clear ();
-
-    TrilinosWrappers::BlockSparsityPattern sp (stokes_partitioner);
-
-    Table<2,DoFTools::Coupling> coupling (dim+1, dim+1);
-    for (unsigned int c=0; c<dim+1; ++c)
-      for (unsigned int d=0; d<dim+1; ++d)
-       if (c == d)
-         coupling[c][d] = DoFTools::always;
-       else
-         coupling[c][d] = DoFTools::none;
-
-    DoFTools::make_sparsity_pattern (stokes_dof_handler, coupling, sp,
-                                    stokes_constraints, false,
-                                    Utilities::Trilinos::
-                                    get_this_mpi_process(trilinos_communicator));
-    sp.compress();
-
-    stokes_preconditioner_matrix.reinit (sp);
-  }
-
+  
   temperature_partitioner
     = Epetra_Map (n_T,
                  DoFTools::count_dofs_with_subdomain_association
@@ -682,23 +1107,24 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
                   Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
                  0,
                  trilinos_communicator);
-  {
-    T_preconditioner.reset ();
-    temperature_mass_matrix.clear ();
-    temperature_stiffness_matrix.clear ();
-    temperature_matrix.clear ();
-
-    TrilinosWrappers::SparsityPattern sp (temperature_partitioner);
-    DoFTools::make_sparsity_pattern (temperature_dof_handler, sp,
-                                    temperature_constraints, false,
-                                    Utilities::Trilinos::
-                                    get_this_mpi_process(trilinos_communicator));
-    sp.compress();
-
-    temperature_matrix.reinit (sp);
-    temperature_mass_matrix.reinit (sp);
-    temperature_stiffness_matrix.reinit (sp);
-  }
+
+  if (Utilities::Trilinos::get_n_mpi_processes(trilinos_communicator) == 1)
+    {
+      Threads::TaskGroup<> tasks;
+      tasks += Threads::new_task (&BoussinesqFlowProblem<dim>::setup_stokes_matrix,
+                                 *this);
+      tasks += Threads::new_task (&BoussinesqFlowProblem<dim>::setup_stokes_preconditioner,
+                                 *this);
+      tasks += Threads::new_task (&BoussinesqFlowProblem<dim>::setup_temperature_matrices,
+                                 *this);
+      tasks.join_all ();
+    }
+  else
+    {
+      setup_stokes_matrix ();
+      setup_stokes_preconditioner ();
+      setup_temperature_matrices ();
+    } 
 
   stokes_solution.reinit (stokes_partitioner);
   old_stokes_solution.reinit (stokes_partitioner);
@@ -716,68 +1142,91 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
 
 template <int dim>
 void
-BoussinesqFlowProblem<dim>::assemble_stokes_preconditioner ()
+BoussinesqFlowProblem<dim>::
+local_assemble_stokes_preconditioner (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                     Assembly::Scratch::StokesPreconditioner<dim> &scratch,
+                                     Assembly::CopyData::StokesPreconditioner<dim> &data)
 {
-  stokes_preconditioner_matrix = 0;
-
-  const QGauss<dim> quadrature_formula (stokes_degree+2);
-  FEValues<dim>     stokes_fe_values (stokes_fe, quadrature_formula,
-                                     update_JxW_values |
-                                     update_values |
-                                     update_gradients);
   const unsigned int   dofs_per_cell   = stokes_fe.dofs_per_cell;
+  const unsigned int   n_q_points      = scratch.stokes_fe_values.n_quadrature_points;
 
-  const unsigned int   n_q_points      = quadrature_formula.size();
+  const FEValuesExtractors::Vector velocities (0);
+  const FEValuesExtractors::Scalar pressure (dim);
 
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
+  scratch.stokes_fe_values.reinit (cell);
+  cell->get_dof_indices (data.local_dof_indices);
 
-  std::vector<Tensor<2,dim> > phi_grad_u (dofs_per_cell);
-  std::vector<double>         phi_p      (dofs_per_cell);
+  data.local_matrix = 0;
 
-  const FEValuesExtractors::Vector velocities (0);
-  const FEValuesExtractors::Scalar pressure (dim);
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
+       {
+         scratch.grad_phi_u[k] = scratch.stokes_fe_values[velocities].gradient(k,q);
+         scratch.phi_p[k]      = scratch.stokes_fe_values[pressure].value (k, q);
+       }
+         
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       for (unsigned int j=0; j<dofs_per_cell; ++j)
+         data.local_matrix(i,j) += (EquationData::eta *
+                                    scalar_product (scratch.grad_phi_u[i], scratch.grad_phi_u[j])
+                                    +
+                                    (1./EquationData::eta) *
+                                    scratch.phi_p[i] * scratch.phi_p[j])
+                                   * scratch.stokes_fe_values.JxW(q);
+    }
+}
 
-  typename DoFHandler<dim>::active_cell_iterator
-    cell = stokes_dof_handler.begin_active(),
-    endc = stokes_dof_handler.end();
-  for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == 
-       Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
-      {
-       stokes_fe_values.reinit (cell);
 
-                                  // only need to recalculate local matrix
-                                  // if FEValues data has changed.
-       if (stokes_fe_values.get_cell_similarity() != CellSimilarity::translation)
-         {
-           local_matrix = 0;
 
-           for (unsigned int q=0; q<n_q_points; ++q)
-             {
-               for (unsigned int k=0; k<dofs_per_cell; ++k)
-                 {
-                   phi_grad_u[k] = stokes_fe_values[velocities].gradient(k,q);
-                   phi_p[k]      = stokes_fe_values[pressure].value (k, q);
-                 }
-
-               for (unsigned int i=0; i<dofs_per_cell; ++i)
-                 for (unsigned int j=0; j<dofs_per_cell; ++j)
-                   local_matrix(i,j) += (EquationData::eta *
-                                         scalar_product (phi_grad_u[i], phi_grad_u[j])
-                                         +
-                                         (1./EquationData::eta) *
-                                         phi_p[i] * phi_p[j])
-                                        * stokes_fe_values.JxW(q);
-             }
-         }
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_stokes_preconditioner (const Assembly::CopyData::StokesPreconditioner<dim> &data)
+{
+  stokes_constraints.distribute_local_to_global (data.local_matrix,
+                                                data.local_dof_indices,
+                                                stokes_preconditioner_matrix);
+}
 
-       cell->get_dof_indices (local_dof_indices);
-       stokes_constraints.distribute_local_to_global (local_matrix,
-                                                      local_dof_indices,
-                                                      stokes_preconditioner_matrix);
-      }
 
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::assemble_stokes_preconditioner ()
+{
+  stokes_preconditioner_matrix = 0;
+
+  const QGauss<dim> quadrature_formula(stokes_degree+2);
+
+  typedef
+    FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>
+    SubdomainFilter;
+  
+  WorkStream::
+    run (SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          stokes_dof_handler.begin_active()),
+        SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          stokes_dof_handler.end()),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          local_assemble_stokes_preconditioner,
+                          this,
+                          _1,
+                          _2,
+                          _3),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          copy_local_to_global_stokes_preconditioner,
+                          this,
+                          _1),
+         Assembly::Scratch::
+         StokesPreconditioner<dim> (stokes_fe, quadrature_formula,
+                                    update_JxW_values |
+                                    update_values |
+                                    update_gradients),
+         Assembly::CopyData::
+        StokesPreconditioner<dim> (stokes_fe));
+  
   stokes_preconditioner_matrix.compress();
 }
 
@@ -795,29 +1244,27 @@ BoussinesqFlowProblem<dim>::build_stokes_preconditioner ()
 
   assemble_stokes_preconditioner ();
 
-  Amg_preconditioner = std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionAMG>
-                      (new TrilinosWrappers::PreconditionAMG());
-
   std::vector<std::vector<bool> > constant_modes;
   std::vector<bool>  velocity_components (dim+1,true);
   velocity_components[dim] = false;
   DoFTools::extract_constant_modes (stokes_dof_handler, velocity_components, 
                                    constant_modes);
 
-  TrilinosWrappers::PreconditionAMG::AdditionalData amg_data;
-  amg_data.constant_modes = constant_modes;
-  amg_data.elliptic = true;
-  amg_data.higher_order_elements = true;
-  amg_data.smoother_sweeps = 2;
-  amg_data.aggregation_threshold = 0.02;
-
-  Amg_preconditioner->initialize(stokes_preconditioner_matrix.block(0,0),
-                                amg_data);
+  Mp_preconditioner  = std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU>
+                      (new TrilinosWrappers::PreconditionILU());
+  Amg_preconditioner = std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionAMG>
+                      (new TrilinosWrappers::PreconditionAMG());
 
-  Mp_preconditioner = std_cxx1x::shared_ptr<TrilinosWrappers::PreconditionILU>
-                                   (new TrilinosWrappers::PreconditionILU());
+  TrilinosWrappers::PreconditionAMG::AdditionalData Amg_data;
+  Amg_data.constant_modes = constant_modes;
+  Amg_data.elliptic = true;
+  Amg_data.higher_order_elements = true;
+  Amg_data.smoother_sweeps = 2;
+  Amg_data.aggregation_threshold = 0.02;
 
   Mp_preconditioner->initialize (stokes_preconditioner_matrix.block(1,1));
+  Amg_preconditioner->initialize (stokes_preconditioner_matrix.block(0,0),
+                                 Amg_data);
 
   rebuild_stokes_preconditioner = false;
 
@@ -827,7 +1274,92 @@ BoussinesqFlowProblem<dim>::build_stokes_preconditioner ()
 
 
 
-                                // @sect4{BoussinesqFlowProblem::assemble_stokes_system}
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+local_assemble_stokes_system (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                             Assembly::Scratch::StokesSystem<dim> &scratch,
+                             Assembly::CopyData::StokesSystem<dim> &data)
+{
+  const unsigned int dofs_per_cell = scratch.stokes_fe_values.get_fe().dofs_per_cell;
+  const unsigned int n_q_points    = scratch.stokes_fe_values.n_quadrature_points;
+  
+  const FEValuesExtractors::Vector velocities (0);
+  const FEValuesExtractors::Scalar pressure (dim);
+
+  scratch.stokes_fe_values.reinit (cell);
+
+  typename DoFHandler<dim>::active_cell_iterator
+    temperature_cell (&triangulation,
+                     cell->level(),
+                     cell->index(),
+                     &temperature_dof_handler);
+  scratch.temperature_fe_values.reinit (temperature_cell);
+      
+  if (rebuild_stokes_matrix)
+    data.local_matrix = 0;
+  data.local_rhs = 0;
+
+  scratch.temperature_fe_values.get_function_values (old_temperature_solution, 
+                                                    scratch.old_temperature_values);
+
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      const double old_temperature = scratch.old_temperature_values[q];
+
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
+       {
+         scratch.phi_u[k] = scratch.stokes_fe_values[velocities].value (k,q);
+         if (rebuild_stokes_matrix)
+           {
+             scratch.grads_phi_u[k] = scratch.stokes_fe_values[velocities].symmetric_gradient(k,q);
+             scratch.div_phi_u[k]   = scratch.stokes_fe_values[velocities].divergence (k, q);
+             scratch.phi_p[k]       = scratch.stokes_fe_values[pressure].value (k, q);
+           }
+       }
+
+      if (rebuild_stokes_matrix)
+       for (unsigned int i=0; i<dofs_per_cell; ++i)
+         for (unsigned int j=0; j<dofs_per_cell; ++j)
+           data.local_matrix(i,j) += (EquationData::eta * 2 *
+                                      scratch.grads_phi_u[i] * scratch.grads_phi_u[j]
+                                      - scratch.div_phi_u[i] * scratch.phi_p[j]
+                                      - scratch.phi_p[i] * scratch.div_phi_u[j])
+                                     * scratch.stokes_fe_values.JxW(q);
+
+      const Point<dim> gravity = ( (dim == 2) ? (Point<dim> (0,1)) : 
+                                  (Point<dim> (0,0,1)) );
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       data.local_rhs(i) += (EquationData::Rayleigh_number *
+                             gravity * scratch.phi_u[i] * old_temperature)*
+                            scratch.stokes_fe_values.JxW(q);
+    }
+
+  cell->get_dof_indices (data.local_dof_indices);
+}
+
+
+
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_stokes_system (const Assembly::CopyData::StokesSystem<dim> &data)
+{
+  if (rebuild_stokes_matrix == true)
+    stokes_constraints.distribute_local_to_global (data.local_matrix,
+                                                  data.local_rhs,
+                                                  data.local_dof_indices,
+                                                  stokes_matrix,
+                                                  stokes_rhs);
+  else
+    stokes_constraints.distribute_local_to_global (data.local_rhs,
+                                                  data.local_dof_indices,
+                                                  stokes_rhs);
+}
+
+
+
+// @sect4{BoussinesqFlowProblem::assemble_stokes_system}
 template <int dim>
 void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
 {
@@ -841,121 +1373,113 @@ void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
   stokes_rhs=0;
 
   const QGauss<dim> quadrature_formula(stokes_degree+2);
-  FEValues<dim>     stokes_fe_values (stokes_fe, quadrature_formula,
-                                     update_values    |
-                                     update_quadrature_points  |
-                                     update_JxW_values |
-                                     (rebuild_stokes_matrix == true
-                                      ?
-                                      update_gradients
-                                      :
-                                      UpdateFlags(0)));
+
+  typedef
+    FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>
+    SubdomainFilter;
+
+  WorkStream::
+    run (SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          stokes_dof_handler.begin_active()),
+         SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          stokes_dof_handler.end()),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          local_assemble_stokes_system,
+                          this,
+                          _1,
+                          _2,
+                          _3),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          copy_local_to_global_stokes_system,
+                          this,
+                          _1),
+         Assembly::Scratch::
+         StokesSystem<dim> (stokes_fe, quadrature_formula,
+                            (update_values    |
+                             update_quadrature_points  |
+                             update_JxW_values |
+                             (rebuild_stokes_matrix == true
+                              ?
+                              update_gradients
+                              :
+                              UpdateFlags(0))),
+                            temperature_fe,
+                            update_values),
+         Assembly::CopyData::
+        StokesSystem<dim> (stokes_fe));
   
-  FEValues<dim>     temperature_fe_values (temperature_fe, quadrature_formula,
-                                          update_values);
+  stokes_matrix.compress();
+  stokes_rhs.compress();
 
-  const unsigned int   dofs_per_cell   = stokes_fe.dofs_per_cell;
+  rebuild_stokes_matrix = false;
+
+  pcout << std::endl;
+  computing_timer.exit_section();
+}
 
-  const unsigned int   n_q_points      = quadrature_formula.size();
 
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
-  Vector<double>       local_rhs (dofs_per_cell);
 
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
 
-  std::vector<double>               old_temperature_values(n_q_points);
 
-  std::vector<Tensor<1,dim> >          phi_u       (dofs_per_cell);
-  std::vector<SymmetricTensor<2,dim> > grads_phi_u (dofs_per_cell);
-  std::vector<double>                  div_phi_u   (dofs_per_cell);
-  std::vector<double>                  phi_p       (dofs_per_cell);
 
-  const FEValuesExtractors::Vector velocities (0);
-  const FEValuesExtractors::Scalar pressure (dim);
+                                // @sect4{BoussinesqFlowProblem::assemble_temperature_system}
+template <int dim>
+void BoussinesqFlowProblem<dim>::
+local_assemble_temperature_matrix (const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                  Assembly::Scratch::TemperatureMatrix<dim> &scratch,
+                                  Assembly::CopyData::TemperatureMatrix<dim> &data)
+{
+  const unsigned int dofs_per_cell = scratch.temperature_fe_values.get_fe().dofs_per_cell;
+  const unsigned int n_q_points    = scratch.temperature_fe_values.n_quadrature_points;
 
-  typename DoFHandler<dim>::active_cell_iterator
-    cell = stokes_dof_handler.begin_active(),
-    endc = stokes_dof_handler.end();
-  typename DoFHandler<dim>::active_cell_iterator
-    temperature_cell = temperature_dof_handler.begin_active();
+  scratch.temperature_fe_values.reinit (cell);
+  cell->get_dof_indices (data.local_dof_indices);
+
+  data.local_mass_matrix = 0;
+  data.local_stiffness_matrix = 0;
   
-  for (; cell!=endc; ++cell, ++temperature_cell)
-    if (cell->subdomain_id() == 
-       Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
-      {
-       stokes_fe_values.reinit (cell);
-       temperature_fe_values.reinit (temperature_cell);
        
-       if (stokes_fe_values.get_cell_similarity() != CellSimilarity::translation)
-         local_matrix = 0;
-       local_rhs = 0;
-  
-       temperature_fe_values.get_function_values (old_temperature_solution, 
-                                                  old_temperature_values);
-  
-       for (unsigned int q=0; q<n_q_points; ++q)
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
+       {
+         scratch.grad_phi_T[k] = scratch.temperature_fe_values.shape_grad (k,q);
+         scratch.phi_T[k]      = scratch.temperature_fe_values.shape_value (k, q);
+       }
+           
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       for (unsigned int j=0; j<dofs_per_cell; ++j)
          {
-           const double old_temperature = old_temperature_values[q];
-  
-           for (unsigned int k=0; k<dofs_per_cell; ++k)
-             {
-               phi_u[k] = stokes_fe_values[velocities].value (k,q);
-               if (rebuild_stokes_matrix && 
-                   stokes_fe_values.get_cell_similarity() != CellSimilarity::translation)
-                 {
-                   grads_phi_u[k] = stokes_fe_values[velocities].symmetric_gradient(k,q);
-                   div_phi_u[k]   = stokes_fe_values[velocities].divergence (k, q);
-                   phi_p[k]       = stokes_fe_values[pressure].value (k, q);
-                 }
-             }
-  
-           if (rebuild_stokes_matrix &&
-               stokes_fe_values.get_cell_similarity() != CellSimilarity::translation)
-             for (unsigned int i=0; i<dofs_per_cell; ++i)
-               for (unsigned int j=0; j<dofs_per_cell; ++j)
-                 local_matrix(i,j) += (EquationData::eta * 2 *
-                                       grads_phi_u[i] * grads_phi_u[j]
-                                       - div_phi_u[i] * phi_p[j]
-                                       - phi_p[i] * div_phi_u[j])
-                                      * stokes_fe_values.JxW(q);
-
-           const Point<dim> gravity = ( (dim == 2) ? (Point<dim> (0,1)) : 
-                                        (Point<dim> (0,0,1)) );
-           for (unsigned int i=0; i<dofs_per_cell; ++i)
-             local_rhs(i) += (EquationData::Rayleigh_number *
-                              gravity * phi_u[i] * old_temperature)*
-                             stokes_fe_values.JxW(q);
+           data.local_mass_matrix(i,j)
+             += (scratch.phi_T[i] * scratch.phi_T[j]
+                 *
+                 scratch.temperature_fe_values.JxW(q));
+           data.local_stiffness_matrix(i,j)
+             += (EquationData::kappa * scratch.grad_phi_T[i] * scratch.grad_phi_T[j]
+                 *
+                 scratch.temperature_fe_values.JxW(q));
          }
-  
-       cell->get_dof_indices (local_dof_indices);
-  
-       if (rebuild_stokes_matrix == true)
-         stokes_constraints.distribute_local_to_global (local_matrix,
-                                                        local_rhs,
-                                                        local_dof_indices,
-                                                        stokes_matrix,
-                                                        stokes_rhs);
-       else
-         stokes_constraints.distribute_local_to_global (local_rhs,
-                                                        local_dof_indices,
-                                                        stokes_rhs);
-      }
-
-  stokes_matrix.compress();
-  stokes_rhs.compress();
-
-  rebuild_stokes_matrix = false;
-
-  pcout << std::endl;
-  computing_timer.exit_section();
+    }
 }
 
 
 
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_temperature_matrix (const Assembly::CopyData::TemperatureMatrix<dim> &data)
+{
+  temperature_constraints.distribute_local_to_global (data.local_mass_matrix,
+                                                     data.local_dof_indices,
+                                                     temperature_mass_matrix);
+  temperature_constraints.distribute_local_to_global (data.local_stiffness_matrix,
+                                                     data.local_dof_indices,
+                                                     temperature_stiffness_matrix);
+}
 
 
-
-                                // @sect4{BoussinesqFlowProblem::assemble_temperature_system}
 template <int dim>
 void BoussinesqFlowProblem<dim>::assemble_temperature_matrix ()
 {
@@ -967,80 +1491,177 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_matrix ()
   temperature_stiffness_matrix = 0;
   
   const QGauss<dim> quadrature_formula(temperature_degree+2);
-  FEValues<dim>     temperature_fe_values (temperature_fe, quadrature_formula,
-                                          update_values    | update_gradients |
-                                          update_JxW_values);
 
-  const unsigned int   dofs_per_cell   = temperature_fe.dofs_per_cell;
-  const unsigned int   n_q_points      = quadrature_formula.size();
+  typedef
+    FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>
+    SubdomainFilter;
+
+  WorkStream::
+    run (SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          temperature_dof_handler.begin_active()),
+         SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          temperature_dof_handler.end()),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          local_assemble_temperature_matrix,
+                          this,
+                          _1,
+                          _2,
+                          _3),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          copy_local_to_global_temperature_matrix,
+                          this,
+                          _1),
+         Assembly::Scratch::
+         TemperatureMatrix<dim> (temperature_fe, quadrature_formula),
+         Assembly::CopyData::
+        TemperatureMatrix<dim> (temperature_fe));
+
+  temperature_mass_matrix.compress();
+  temperature_stiffness_matrix.compress();
+
+  rebuild_temperature_matrices = false;
+  rebuild_temperature_preconditioner = true;
+
+  computing_timer.exit_section();
+}
+
+
 
-  FullMatrix<double>   local_mass_matrix (dofs_per_cell, dofs_per_cell);
-  FullMatrix<double>   local_stiffness_matrix (dofs_per_cell, dofs_per_cell);
+template <int dim>
+void BoussinesqFlowProblem<dim>::
+local_assemble_temperature_rhs (const std::pair<double,double> global_T_range,
+                               const double                   global_max_velocity,
+                               const typename DoFHandler<dim>::active_cell_iterator &cell,
+                               Assembly::Scratch::TemperatureRHS<dim> &scratch,
+                               Assembly::CopyData::TemperatureRHS<dim> &data)
+{
+  const bool use_bdf2_scheme = (timestep_number != 0);
 
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
+  const unsigned int dofs_per_cell = scratch.temperature_fe_values.get_fe().dofs_per_cell;
+  const unsigned int n_q_points    = scratch.temperature_fe_values.n_quadrature_points;
 
-  std::vector<double>                  phi_T       (dofs_per_cell);
-  std::vector<Tensor<1,dim> >          grad_phi_T  (dofs_per_cell);
+  EquationData::TemperatureRightHandSide<dim>  temperature_right_hand_side;
+
+  const FEValuesExtractors::Vector velocities (0);
+  
+  data.local_rhs = 0;
+  
+  scratch.temperature_fe_values.reinit (cell);
 
   typename DoFHandler<dim>::active_cell_iterator
-    cell = temperature_dof_handler.begin_active(),
-    endc = temperature_dof_handler.end();
-  for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == 
-       Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) )
-      {
-       temperature_fe_values.reinit (cell);
-       
-       if (temperature_fe_values.get_cell_similarity() != CellSimilarity::translation)
-         {
-           local_mass_matrix = 0;
-           local_stiffness_matrix = 0;
+    stokes_cell (&triangulation,
+                cell->level(),
+                cell->index(),
+                &stokes_dof_handler);
+  scratch.stokes_fe_values.reinit (stokes_cell);
   
-           for (unsigned int q=0; q<n_q_points; ++q)
-             {
-               for (unsigned int k=0; k<dofs_per_cell; ++k)
-                 {
-                   grad_phi_T[k] = temperature_fe_values.shape_grad (k,q);
-                   phi_T[k]      = temperature_fe_values.shape_value (k, q);
-                 }
-
-               for (unsigned int i=0; i<dofs_per_cell; ++i)
-                 for (unsigned int j=0; j<dofs_per_cell; ++j)
-                   {
-                     local_mass_matrix(i,j)
-                       += (phi_T[i] * phi_T[j]
-                           *
-                           temperature_fe_values.JxW(q));
-                     local_stiffness_matrix(i,j)
-                       += (EquationData::kappa * grad_phi_T[i] * grad_phi_T[j]
-                           *
-                           temperature_fe_values.JxW(q));
-                   }
-             }
-         }
+  scratch.temperature_fe_values.get_function_values (old_temperature_solution,
+                                                    scratch.old_temperature_values);
+  scratch.temperature_fe_values.get_function_values (old_old_temperature_solution,
+                                                    scratch.old_old_temperature_values);
+  
+  scratch.temperature_fe_values.get_function_gradients (old_temperature_solution,
+                                                       scratch.old_temperature_grads);
+  scratch.temperature_fe_values.get_function_gradients (old_old_temperature_solution,
+                                                       scratch.old_old_temperature_grads);
+       
+  scratch.temperature_fe_values.get_function_laplacians (old_temperature_solution,
+                                                        scratch.old_temperature_laplacians);
+  scratch.temperature_fe_values.get_function_laplacians (old_old_temperature_solution,
+                                                        scratch.old_old_temperature_laplacians);
+       
+  temperature_right_hand_side.value_list (scratch.temperature_fe_values.get_quadrature_points(),
+                                         scratch.gamma_values);
+
+  scratch.stokes_fe_values[velocities].get_function_values (stokes_solution,
+                                                           scratch.old_velocity_values);
+  scratch.stokes_fe_values[velocities].get_function_values (old_stokes_solution,
+                                                           scratch.old_old_velocity_values);
        
-       cell->get_dof_indices (local_dof_indices);
+  const double nu
+    = compute_viscosity (scratch.old_temperature_values,
+                        scratch.old_old_temperature_values,
+                        scratch.old_temperature_grads,
+                        scratch.old_old_temperature_grads,
+                        scratch.old_temperature_laplacians,
+                        scratch.old_old_temperature_laplacians,
+                        scratch.old_velocity_values,
+                        scratch.old_old_velocity_values,
+                        scratch.gamma_values,
+                        global_max_velocity,
+                        global_T_range.second - global_T_range.first,
+                        cell->diameter());
+       
+  for (unsigned int q=0; q<n_q_points; ++q)
+    {
+      for (unsigned int k=0; k<dofs_per_cell; ++k)
+       {
+         scratch.phi_T[k]      = scratch.temperature_fe_values.shape_value (k, q);
+         scratch.grad_phi_T[k] = scratch.temperature_fe_values.shape_grad (k,q);
+       }
   
-       temperature_constraints.distribute_local_to_global (local_mass_matrix,
-                                                           local_dof_indices,
-                                                           temperature_mass_matrix);
-       temperature_constraints.distribute_local_to_global (local_stiffness_matrix,
-                                                           local_dof_indices,
-                                                           temperature_stiffness_matrix);
-      }
 
-  temperature_mass_matrix.compress();
-  temperature_stiffness_matrix.compress();
+      const double old_Ts
+       = (use_bdf2_scheme ?
+          (scratch.old_temperature_values[q] *
+           (time_step + old_time_step) / old_time_step
+           -
+           scratch.old_old_temperature_values[q] *
+           (time_step * time_step) /
+           (old_time_step * (time_step + old_time_step)))
+          :
+          scratch.old_temperature_values[q]);
+
+      const Tensor<1,dim> ext_grad_T
+       = (use_bdf2_scheme ? 
+          (scratch.old_temperature_grads[q] *
+           (1+time_step/old_time_step) 
+           -
+           scratch.old_old_temperature_grads[q] *
+           time_step / old_time_step) 
+          :
+          scratch.old_temperature_grads[q]);
+         
+      const Tensor<1,dim> extrapolated_u
+       = (use_bdf2_scheme ? 
+          (scratch.old_velocity_values[q] * (1+time_step/old_time_step) - 
+           scratch.old_old_velocity_values[q] * time_step/old_time_step)
+          :
+          scratch.old_velocity_values[q]);
+
+      for (unsigned int i=0; i<dofs_per_cell; ++i)
+       data.local_rhs(i) += (old_Ts * scratch.phi_T[i]
+                             -
+                             time_step *
+                             extrapolated_u * ext_grad_T * scratch.phi_T[i]
+                             -
+                             time_step *
+                             nu * ext_grad_T * scratch.grad_phi_T[i]
+                             +
+                             time_step *
+                             scratch.gamma_values[q] * scratch.phi_T[i])
+                            *
+                            scratch.temperature_fe_values.JxW(q);
+    }
+       
+  cell->get_dof_indices (data.local_dof_indices);
+}
 
-  rebuild_temperature_matrices = false;
-  rebuild_temperature_preconditioner = true;
 
-  computing_timer.exit_section();
+template <int dim>
+void
+BoussinesqFlowProblem<dim>::
+copy_local_to_global_temperature_rhs (const Assembly::CopyData::TemperatureRHS<dim> &data)
+{
+  temperature_constraints.distribute_local_to_global (data.local_rhs,
+                                                     data.local_dof_indices,
+                                                     temperature_rhs);
 }
 
 
 
-
 template <int dim>
 void BoussinesqFlowProblem<dim>::assemble_temperature_system (const double maximal_velocity)
 {
@@ -1072,159 +1693,36 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_system (const double maxim
   temperature_rhs = 0;
   
   const QGauss<dim> quadrature_formula(temperature_degree+2);
-  FEValues<dim>     temperature_fe_values (temperature_fe, quadrature_formula,
-                                          update_values    |
-                                          update_gradients |
-                                          update_hessians  |
-                                          update_quadrature_points |
-                                          update_JxW_values);
-  FEValues<dim> stokes_fe_values (stokes_fe, quadrature_formula,
-                                 update_values);
-
-  const unsigned int   dofs_per_cell   = temperature_fe.dofs_per_cell;
-  const unsigned int   n_q_points      = quadrature_formula.size();
-
-  Vector<double>       local_rhs (dofs_per_cell);
-  FullMatrix<double>   local_matrix (dofs_per_cell, dofs_per_cell);
-
-  std::vector<unsigned int> local_dof_indices (dofs_per_cell);
-
-  std::vector<Tensor<1,dim> > old_velocity_values (n_q_points);
-  std::vector<Tensor<1,dim> > old_old_velocity_values (n_q_points);
-  
-  std::vector<double>         old_temperature_values (n_q_points);
-  std::vector<double>         old_old_temperature_values(n_q_points);
-  std::vector<Tensor<1,dim> > old_temperature_grads(n_q_points);
-  std::vector<Tensor<1,dim> > old_old_temperature_grads(n_q_points);
-  std::vector<double>         old_temperature_laplacians(n_q_points);
-  std::vector<double>         old_old_temperature_laplacians(n_q_points);
-
-  
-  EquationData::TemperatureRightHandSide<dim>  temperature_right_hand_side;
-  std::vector<double> gamma_values (n_q_points);
-
-  std::vector<double>                  phi_T       (dofs_per_cell);
-  std::vector<Tensor<1,dim> >          grad_phi_T  (dofs_per_cell);
-  
   const std::pair<double,double>
     global_T_range = get_extrapolated_temperature_range();
 
-  const TrilinosWrappers::BlockVector 
-    localized_stokes_solution (stokes_solution);
-  const TrilinosWrappers::BlockVector 
-    localized_old_stokes_solution (old_stokes_solution);
-
-  const FEValuesExtractors::Vector velocities (0);
-  
-  typename DoFHandler<dim>::active_cell_iterator
-    cell = temperature_dof_handler.begin_active(),
-    endc = temperature_dof_handler.end();
-  typename DoFHandler<dim>::active_cell_iterator
-    stokes_cell = stokes_dof_handler.begin_active();
-
-  for (; cell!=endc; ++cell, ++stokes_cell)
-    if (cell->subdomain_id() == 
-       Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) )
-      {
-       local_rhs = 0;
-  
-       temperature_fe_values.reinit (cell);
-       stokes_fe_values.reinit (stokes_cell);
-  
-       temperature_fe_values.get_function_values (old_temperature_solution,
-                                                  old_temperature_values);
-       temperature_fe_values.get_function_values (old_old_temperature_solution,
-                                                  old_old_temperature_values);
-  
-       temperature_fe_values.get_function_gradients (old_temperature_solution,
-                                                     old_temperature_grads);
-       temperature_fe_values.get_function_gradients (old_old_temperature_solution,
-                                                     old_old_temperature_grads);
-       
-       temperature_fe_values.get_function_laplacians (old_temperature_solution,
-                                                      old_temperature_laplacians);
-       temperature_fe_values.get_function_laplacians (old_old_temperature_solution,
-                                                      old_old_temperature_laplacians);
-       
-       temperature_right_hand_side.value_list (temperature_fe_values.get_quadrature_points(),
-                                               gamma_values);
-
-       stokes_fe_values[velocities].get_function_values (localized_stokes_solution,
-                                                         old_velocity_values);
-       stokes_fe_values[velocities].get_function_values (localized_old_stokes_solution,
-                                                         old_old_velocity_values);
-       
-       const double nu
-         = compute_viscosity (old_temperature_values,
-                              old_old_temperature_values,
-                              old_temperature_grads,
-                              old_old_temperature_grads,
-                              old_temperature_laplacians,
-                              old_old_temperature_laplacians,
-                              old_velocity_values,
-                              old_old_velocity_values,
-                              gamma_values,
-                              maximal_velocity,
-                              global_T_range.second - global_T_range.first,
-                              cell->diameter());
-       
-       for (unsigned int q=0; q<n_q_points; ++q)
-         {
-           for (unsigned int k=0; k<dofs_per_cell; ++k)
-             {
-               grad_phi_T[k] = temperature_fe_values.shape_grad (k,q);
-               phi_T[k]      = temperature_fe_values.shape_value (k, q);
-             }
-  
-
-         const double old_Ts
-           = (use_bdf2_scheme ?
-              (old_temperature_values[q] *
-               (time_step + old_time_step) / old_time_step
-               -
-               old_old_temperature_values[q] *
-               (time_step * time_step) /
-               (old_time_step * (time_step + old_time_step)))
-              :
-              old_temperature_values[q]);
-
-         const Tensor<1,dim> ext_grad_T
-           = (use_bdf2_scheme ? 
-              (old_temperature_grads[q] *
-               (1+time_step/old_time_step) 
-               -
-               old_old_temperature_grads[q] *
-               time_step / old_time_step) 
-              :
-              old_temperature_grads[q]);
-         
-         const Tensor<1,dim> extrapolated_u
-           = (use_bdf2_scheme ? 
-              (old_velocity_values[q] * (1+time_step/old_time_step) - 
-               old_old_velocity_values[q] * time_step/old_time_step)
-              :
-              old_velocity_values[q]);
-
-         for (unsigned int i=0; i<dofs_per_cell; ++i)
-           local_rhs(i) += (old_Ts * phi_T[i]
-                            -
-                            time_step *
-                            extrapolated_u * ext_grad_T * phi_T[i]
-                            -
-                            time_step *
-                            nu * ext_grad_T * grad_phi_T[i]
-                            +
-                            time_step *
-                            gamma_values[q] * phi_T[i])
-                           *
-                           temperature_fe_values.JxW(q);
-         }
-       
-       cell->get_dof_indices (local_dof_indices);
-       temperature_constraints.distribute_local_to_global (local_rhs,
-                                                           local_dof_indices,
-                                                           temperature_rhs);
-      }
+  typedef
+    FilteredIterator<typename DoFHandler<dim>::active_cell_iterator>
+    SubdomainFilter;
+
+  WorkStream::
+    run (SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          temperature_dof_handler.begin_active()),
+         SubdomainFilter (IteratorFilters::SubdomainEqualTo
+                          (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
+                          temperature_dof_handler.end()),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          local_assemble_temperature_rhs,
+                          this,
+                          global_T_range,
+                          maximal_velocity,
+                          _1,
+                          _2,
+                          _3),
+         std_cxx1x::bind (&BoussinesqFlowProblem<dim>::
+                          copy_local_to_global_temperature_rhs,
+                          this,
+                          _1),
+         Assembly::Scratch::
+         TemperatureRHS<dim> (temperature_fe, stokes_fe, quadrature_formula),
+         Assembly::CopyData::
+        TemperatureRHS<dim> (temperature_fe));
 
   temperature_rhs.compress();
 }
@@ -1320,29 +1818,35 @@ void BoussinesqFlowProblem<dim>::solve ()
     SolverControl solver_control (stokes_matrix.m(),
                                  1e-6*stokes_rhs.l2_norm());
 
-    SolverGMRES<TrilinosWrappers::MPI::BlockVector>
-      gmres (solver_control,
-            SolverGMRES<TrilinosWrappers::MPI::BlockVector >::AdditionalData(100));
+    SolverBicgstab<TrilinosWrappers::MPI::BlockVector>
+      bicgstab (solver_control, false);
+
+    TrilinosWrappers::MPI::BlockVector
+      distributed_stokes_solution (stokes_partitioner);
+    distributed_stokes_solution = stokes_solution;
 
                                   // now treat the hanging nodes correctly.
-    const unsigned int start = stokes_solution.block(1).local_range().first + 
-      stokes_solution.block(0).size();
-    const unsigned int end = stokes_solution.block(1).local_range().second + 
-      stokes_solution.block(0).size();
+    const unsigned int start = 
+      distributed_stokes_solution.block(1).local_range().first + 
+      distributed_stokes_solution.block(0).size();
+    const unsigned int end = 
+      distributed_stokes_solution.block(1).local_range().second + 
+      distributed_stokes_solution.block(0).size();
     for (unsigned int i=start; i<end; ++i)
       if (stokes_constraints.is_constrained (i))
-       stokes_solution(i) = 0;
-
-    gmres.solve(stokes_matrix, stokes_solution, stokes_rhs, preconditioner);
+       distributed_stokes_solution(i) = 0;
+    
+    bicgstab.solve(stokes_matrix, distributed_stokes_solution, stokes_rhs, 
+                  preconditioner);
 
+    stokes_solution = distributed_stokes_solution;
+    
     pcout << "   "
          << solver_control.last_step()
-         << " GMRES iterations for Stokes subsystem."
+         << " BiCGStab iterations for Stokes subsystem."
          << std::endl;
 
-    TrilinosWrappers::BlockVector localized_stokes_solution (stokes_solution);
-    stokes_constraints.distribute (localized_stokes_solution);
-    stokes_solution = localized_stokes_solution;
+    stokes_constraints.distribute (stokes_solution);
   }
   computing_timer.exit_section();
 
@@ -1372,26 +1876,29 @@ void BoussinesqFlowProblem<dim>::solve ()
                                  1e-8*temperature_rhs.l2_norm());
     SolverCG<TrilinosWrappers::MPI::Vector>   cg (solver_control);
 
-    cg.solve (temperature_matrix, temperature_solution,
+    TrilinosWrappers::MPI::Vector
+      distributed_temperature_solution (temperature_partitioner);
+    distributed_temperature_solution = temperature_solution;
+
+    cg.solve (temperature_matrix, distributed_temperature_solution,
              temperature_rhs, *T_preconditioner);
 
-    TrilinosWrappers::Vector localized_temperature_solution (temperature_solution);
-    temperature_constraints.distribute (localized_temperature_solution);
-    temperature_solution = localized_temperature_solution;
+    temperature_solution = distributed_temperature_solution;
+    temperature_constraints.distribute (temperature_solution);
 
     pcout << "   "
          << solver_control.last_step()
          << " CG iterations for temperature" << std::endl;
     computing_timer.exit_section();
 
-    double min_temperature = localized_temperature_solution(0),
-          max_temperature = localized_temperature_solution(0);
+    double min_temperature = temperature_solution(0),
+          max_temperature = temperature_solution(0);
     for (unsigned int i=1; i<temperature_solution.size(); ++i)
       {
        min_temperature = std::min<double> (min_temperature,
-                                           localized_temperature_solution(i));
+                                           temperature_solution(i));
        max_temperature = std::max<double> (max_temperature,
-                                           localized_temperature_solution(i));
+                                           temperature_solution(i));
       }
     
     pcout << "   Temperature range: "
@@ -1418,8 +1925,6 @@ void BoussinesqFlowProblem<dim>::output_results ()  const
          ExcInternalError());
 
   Vector<double> joint_solution (joint_dof_handler.n_dofs());
-  TrilinosWrappers::BlockVector localized_stokes_solution (stokes_solution);
-  TrilinosWrappers::Vector localized_temperature_solution (temperature_solution);
 
   if (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) == 0)
     {
@@ -1448,7 +1953,7 @@ void BoussinesqFlowProblem<dim>::output_results ()  const
                          local_stokes_dof_indices.size(),
                          ExcInternalError());
                  joint_solution(local_joint_dof_indices[i])
-                   = localized_stokes_solution(local_stokes_dof_indices[joint_fe.system_to_base_index(i).second]);
+                   = stokes_solution(local_stokes_dof_indices[joint_fe.system_to_base_index(i).second]);
                }
              else
                {
@@ -1459,7 +1964,7 @@ void BoussinesqFlowProblem<dim>::output_results ()  const
                          local_stokes_dof_indices.size(),
                          ExcInternalError());
                  joint_solution(local_joint_dof_indices[i])
-                   = localized_temperature_solution(local_temperature_dof_indices[joint_fe.system_to_base_index(i).second]);
+                   = temperature_solution(local_temperature_dof_indices[joint_fe.system_to_base_index(i).second]);
                }
          }
       }
@@ -1501,12 +2006,10 @@ void BoussinesqFlowProblem<dim>::refine_mesh (const unsigned int max_grid_level)
   computing_timer.enter_section ("Refine mesh structure, part 1");
   Vector<float> estimated_error_per_cell (triangulation.n_active_cells());
 
-  TrilinosWrappers::Vector localized_temperature_solution (temperature_solution);
-
   KellyErrorEstimator<dim>::estimate (temperature_dof_handler,
                                      QGauss<dim-1>(temperature_degree+1),
                                      typename FunctionMap<dim>::type(),
-                                     localized_temperature_solution,
+                                     temperature_solution,
                                      estimated_error_per_cell);
 
   GridRefinement::refine_and_coarsen_fixed_fraction (triangulation,
diff --git a/deal.II/examples/step-35/Makefile b/deal.II/examples/step-35/Makefile
new file mode 100644 (file)
index 0000000..5df8a94
--- /dev/null
@@ -0,0 +1,154 @@
+# $Id: Makefile 16563 2008-08-15 16:08:28Z bangerth $
+
+
+# For the small projects Makefile, you basically need to fill in only
+# four fields.
+#
+# The first is the name of the application. It is assumed that the
+# application name is the same as the base file name of the single C++
+# file from which the application is generated.
+target = step-35
+
+# The second field determines whether you want to run your program in
+# debug or optimized mode. The latter is significantly faster, but no
+# run-time checking of parameters and internal states is performed, so
+# you should set this value to `on' while you develop your program,
+# and to `off' when running production computations.
+debug-mode = on
+
+
+# As third field, we need to give the path to the top-level deal.II
+# directory. You need to adjust this to your needs. Since this path is
+# probably the most often needed one in the Makefile internals, it is
+# designated by a single-character variable, since that can be
+# reference using $D only, i.e. without the parentheses that are
+# required for most other parameters, as e.g. in $(target).
+D = ../../
+
+
+# The last field specifies the names of data and other files that
+# shall be deleted when calling `make clean'. Object and backup files,
+# executables and the like are removed anyway. Here, we give a list of
+# files in the various output formats that deal.II supports.
+clean-up-files = *gmv *gnuplot *gpl *eps *pov *vtk
+
+
+
+
+#
+#
+# Usually, you will not need to change anything beyond this point.
+#
+#
+# The next statement tell the `make' program where to find the
+# deal.II top level directory and to include the file with the global
+# settings
+include $D/common/Make.global_options
+
+
+# Since the whole project consists of only one file, we need not
+# consider difficult dependencies. We only have to declare the
+# libraries which we want to link to the object file, and there need
+# to be two sets of libraries: one for the debug mode version of the
+# application and one for the optimized mode. Here we have selected
+# the versions for 2d. Note that the order in which the libraries are
+# given here is important and that your applications won't link
+# properly if they are given in another order.
+#
+# You may need to augment the lists of libraries when compiling your
+# program for other dimensions, or when using third party libraries
+libs.g   = $(lib-deal2-2d.g) \
+          $(lib-lac.g)      \
+           $(lib-base.g)
+libs.o   = $(lib-deal2-2d.o) \
+          $(lib-lac.o)      \
+           $(lib-base.o)
+
+
+# We now use the variable defined above which switch between debug and
+# optimized mode to select the set of libraries to link with. Included
+# in the list of libraries is the name of the object file which we
+# will produce from the single C++ file. Note that by default we use
+# the extension .g.o for object files compiled in debug mode and .o for
+# object files in optimized mode (or whatever the local default on your
+# system is instead of .o).
+ifeq ($(debug-mode),on)
+  libraries = $(target).g.$(OBJEXT) $(libs.g)
+else
+  libraries = $(target).$(OBJEXT) $(libs.o)
+endif
+
+
+# Now comes the first production rule: how to link the single object
+# file produced from the single C++ file into the executable. Since
+# this is the first rule in the Makefile, it is the one `make' selects
+# if you call it without arguments.
+$(target) : $(libraries)
+       @echo ============================ Linking $@
+       @$(CXX) -o $@$(EXEEXT) $^ $(LIBS) $(LDFLAGS)
+
+
+# To make running the application somewhat independent of the actual
+# program name, we usually declare a rule `run' which simply runs the
+# program. You can then run it by typing `make run'. This is also
+# useful if you want to call the executable with arguments which do
+# not change frequently. You may then want to add them to the
+# following rule:
+run: $(target)
+       @echo ============================ Running $<
+       @./$(target)$(EXEEXT)
+
+
+# As a last rule to the `make' program, we define what to do when
+# cleaning up a directory. This usually involves deleting object files
+# and other automatically created files such as the executable itself,
+# backup files, and data files. Since the latter are not usually quite
+# diverse, you needed to declare them at the top of this file.
+clean:
+       -rm -f *.$(OBJEXT) *~ Makefile.dep $(target)$(EXEEXT) $(clean-up-files)
+
+
+# Since we have not yet stated how to make an object file from a C++
+# file, we should do so now. Since the many flags passed to the
+# compiler are usually not of much interest, we suppress the actual
+# command line using the `at' sign in the first column of the rules
+# and write the string indicating what we do instead.
+./%.g.$(OBJEXT) :
+       @echo ==============debug========= $(<F)
+       @$(CXX) $(CXXFLAGS.g) -c $< -o $@
+./%.$(OBJEXT) :
+       @echo ==============optimized===== $(<F)
+       @$(CXX) $(CXXFLAGS.o) -c $< -o $@
+
+
+# The following statement tells make that the rules `run' and `clean'
+# are not expected to produce files of the same name as Makefile rules
+# usually do.
+.PHONY: run clean
+
+
+# Finally there is a rule which you normally need not care much about:
+# since the executable depends on some include files from the library,
+# besides the C++ application file of course, it is necessary to
+# re-generate the executable when one of the files it depends on has
+# changed. The following rule to created a dependency file
+# `Makefile.dep', which `make' uses to determine when to regenerate
+# the executable. This file is automagically remade whenever needed,
+# i.e. whenever one of the cc-/h-files changed. Make detects whether
+# to remake this file upon inclusion at the bottom of this file.
+#
+# If the creation of Makefile.dep fails, blow it away and fail
+Makefile.dep: $(target).cc Makefile \
+              $(shell echo $D/*/include/*/*.h)
+       @echo ============================ Remaking $@
+       @$D/common/scripts/make_dependencies  $(INCLUDE) -B. $(target).cc \
+               > $@ \
+         || (rm -f $@ ; false)
+       @if test -s $@ ; then : else rm $@ ; fi
+
+
+# To make the dependencies known to `make', we finally have to include
+# them:
+include Makefile.dep
+
+
diff --git a/deal.II/examples/step-35/include/EqData.h b/deal.II/examples/step-35/include/EqData.h
new file mode 100644 (file)
index 0000000..d528e5d
--- /dev/null
@@ -0,0 +1,71 @@
+/*
+    Header file for the classes that define the exact solution
+    and driving force.
+
+    These classes are much like any other Function class
+    so we are not going to comment on them
+
+    by Abner Salgado.
+*/
+#ifndef _EQ_DATA_H_
+#define _EQ_DATA_H_
+
+
+/*
+    We want to be able to select which component of vector
+    valued functions we are going to work on
+*/
+#include "../include/MultiComponentFunction.h"
+
+
+// this is dealii
+#include <base/point.h>
+
+
+// We need to define sines and cosines
+#include <cmath>
+
+
+
+// This is ugly, we have to remove it
+const double PI = std::acos( -1. );
+
+
+
+// The Velocity function
+template<int dim> class Velocity: public Multi_Component_Function<dim> {
+  public:
+    Velocity(const double initial_time =0.0);
+    virtual double value(const Point<dim> &p, const unsigned int component = 0) const;
+    virtual Tensor<1,dim> gradient(const Point<dim> &p, const unsigned int component=0) const;
+    virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values,
+                               const unsigned int component = 0 ) const;
+    virtual void gradient_list( const std::vector< Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                  const unsigned int component = 0 ) const;
+};
+
+
+
+// The Pressure function
+template<int dim> class Pressure: public Function<dim>{
+  public:
+    Pressure(const double initial_time = 0.0);
+    virtual double value(const Point<dim> &p, const unsigned int component = 0) const;
+    virtual Tensor<1,dim> gradient(const Point<dim> &p, const unsigned int component=0) const;
+    virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values,
+                              const unsigned int component = 0 ) const;
+    virtual void gradient_list( const std::vector< Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                  const unsigned int component = 0 ) const;
+};
+
+
+
+// The Force function
+template<int dim> class Force: public Multi_Component_Function<dim>{
+  public:
+    Force( const double initial_time =0.0 );
+    virtual double value( const Point<dim> &p, const unsigned int component = 0 ) const;
+    virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values, const unsigned int component = 0 ) const;
+};
+
+#endif
diff --git a/deal.II/examples/step-35/include/FileReader.h b/deal.II/examples/step-35/include/FileReader.h
new file mode 100644 (file)
index 0000000..4047aab
--- /dev/null
@@ -0,0 +1,70 @@
+// This class is supposed to aid in the reading of parameter files
+#ifndef _FILE_READER_H_
+#define _FILE_READER_H_
+
+
+
+#include <base/parameter_handler.h>
+
+
+
+#include <fstream>
+#include <iostream>
+
+
+
+using namespace dealii;
+
+enum Method_Formulation{
+  METHOD_STANDARD,
+  METHOD_ROTATIONAL
+};
+
+class Data_Storage{
+  public:
+    Data_Storage();
+    ~Data_Storage();
+    void read_data( char *filename );
+    void print_usage();
+    void print_status() const;
+    // The data itself
+    //// The type of method we want to use
+    Method_Formulation form;
+    //// physical data
+    double initial_time,
+           final_time,
+           Reynolds;
+    //// Time stepping data
+    double initial_dt,
+           final_dt,
+           dt_decrement;
+    //// Space discretization data
+    unsigned int n_of_global_refines,
+                 pressure_degree;
+    //// Data to solve the velocity
+    unsigned int vel_max_iterations,
+                 vel_Krylov_size,
+                 vel_off_diagonals,
+                 vel_update_prec;
+    double vel_eps,
+           vel_diag_strength;
+    //// Data to solve the projection
+    unsigned int proj_max_iterations,
+                 proj_off_diagonals;
+    double proj_eps,
+           proj_diag_strength;
+    //// Data to do the pressure update step
+    unsigned int pres_max_iterations,
+                 pres_off_diagonals;
+    double pres_eps,
+           pres_diag_strength;
+    //// Verbosity
+    bool verbose;
+    //// Frequency of the outputted data
+    unsigned int output;
+
+  protected:
+    ParameterHandler prm;
+};
+
+#endif
diff --git a/deal.II/examples/step-35/include/MultiComponentFunction.h b/deal.II/examples/step-35/include/MultiComponentFunction.h
new file mode 100644 (file)
index 0000000..22ba293
--- /dev/null
@@ -0,0 +1,51 @@
+/*
+    Header file for Multi Component Functions.
+    The whole purpose of this class to exist is so we do not have to write the function
+        void set_component( const unsigned int d )
+    twice
+
+    by Abner Salgado.
+*/
+#ifndef _MULTI_COMPONENT_FUNCTION_H_
+#define _MULTI_COMPONENT_FUNCTION_H_
+
+
+/*
+  This is basically a workaround for vector valued functions
+  when we just want one of its components
+  so they are derived from the Function class
+*/
+#include <base/function.h>
+
+
+using namespace dealii;
+
+
+/*
+    The only funcitonality this class has is that it provides a common wrapper
+    for vector valued functions for the case when you only want to ask them for
+    one of their components
+*/
+template<int dim> class Multi_Component_Function: public Function<dim> {
+  public:
+    /*
+      Constructor. It does not do anything interesting but set the initial
+      time for the function to evaluate its values
+    */
+    Multi_Component_Function( const double initial_time = 0. );
+    /*
+      This is the whole reason this class exists.
+      It wouldn't seem logical to have this written twice. One for the
+      Velocity function and one for the Force function.
+      So we are instead going to derive these from this class
+      and so adding this functionality of selecting which component you
+      want to give the value of
+    */
+    void set_component(const unsigned int d );
+  protected:
+    // The current component we are working on
+    unsigned int component;
+};
+
+
+#endif
diff --git a/deal.II/examples/step-35/include/NavierStokes.h b/deal.II/examples/step-35/include/NavierStokes.h
new file mode 100644 (file)
index 0000000..13c8bfc
--- /dev/null
@@ -0,0 +1,278 @@
+/*
+    Definition of the Projection Method for constant density
+    Navier Stokes
+    by Abner Salgado.
+*/
+#ifndef _NAVIER_STOKES_H_
+#define _NAVIER_STOKES_H_
+
+
+
+
+/*
+    First we include the local files, where we have the definitions of
+    1.- The exact solution
+    2.- The class that reads runtime parameters from a file
+*/
+#include "EqData.h"
+#include "FileReader.h"
+
+
+
+/*
+    These includes are to get the quadrature
+    and to handle the convergence table
+*/
+#include <base/quadrature_lib.h>
+#include <base/convergence_table.h>
+
+
+
+/*
+    The includes needed to manage multithreading
+*/
+#include <base/multithread_info.h>
+#include <base/thread_management.h>
+
+
+
+/*
+    Grid management includes.
+    Triangulation handler
+    Grid generator & refinement, accessor and iterators
+*/
+#include <grid/tria.h>
+#include <grid/grid_generator.h>
+#include <grid/grid_refinement.h>
+#include <grid/tria_accessor.h>
+#include <grid/tria_iterator.h>
+#include <grid/tria_boundary_lib.h>
+#include <grid/grid_in.h>
+
+
+
+/*
+    Degrees of freedom management includes
+    Handling, accessing, computing useful stuff from and for them and renumbering.
+*/
+#include <dofs/dof_handler.h>
+#include <dofs/dof_accessor.h>
+#include <dofs/dof_tools.h>
+#include <dofs/dof_renumbering.h>
+#include <dofs/dof_constraints.h>
+
+
+
+/*
+    Finite element management includes
+    Definition of finite elements and extraction of info from them
+    and from finite element functions.
+*/
+#include <fe/fe_q.h>
+#include <fe/fe_values.h>
+#include <fe/fe_tools.h>
+
+
+
+/*
+    Linear algebra includes
+    Sparse matrices, vectors, preconditioners and solvers.
+*/
+#include <lac/vector.h>
+#include <lac/sparse_matrix.h>
+#include <lac/solver_cg.h>
+#include <lac/precondition.h>
+#include <lac/solver_gmres.h>
+#include <lac/sparse_ilu.h>
+#include <lac/sparse_direct.h>
+
+
+
+/*
+    Numerical algorithms includes
+    Assembly of right hand sides, computation of errors, output of the solution, etc
+*/
+#include <numerics/matrices.h>
+#include <numerics/vectors.h>
+#include <numerics/data_out.h>
+
+
+/*
+    This class is the core of all the program
+    It is the implementation of the projection method for the Navier-Stokes equations.
+*/
+template<int dim> class Navier_Stokes_Projection{
+  public:
+    /*
+      Constructor. It takes as a parameter a reference to a Data_Storage class, which is
+      basically a container of all the data this class needs to be defined (and some other stuff)
+      Here we basically just copy the needed data and/or assign the values that are needed.
+    */
+    Navier_Stokes_Projection( const Data_Storage &data );
+    /*
+      Destructor. Completely trivial.
+    */
+    ~Navier_Stokes_Projection();
+    /*
+      This function creates the triangulation and refines it the required number of times
+      It also initializes all the quantities that are mesh dependent but not time stepping dependent
+      i.e. the size of matrices and vectors.
+    */
+    void Create_Triangulation( const unsigned int n_of_refines );
+    /*
+      Having created a mesh and once the time step is set, the next step is to initialize the method
+      i.e. compute the matrices that are never going to change, load the initial data, etc.
+      This is what this function does.
+    */
+    void Initialize();
+    /*
+      This is the time marching function, which starting at t_0 advances in time using the projection method
+      with time step dt until T.
+      The boolean parameter that it takes is to enable information about what the function is doing at the present
+      moment, i.e. diffusion, projection substep; updating preconditioners etc.
+      This is useful mostly for debuggin purposes and so it is by default set to false
+    */
+    void run( const bool verbose = false, const unsigned int n_of_plots = 10 );
+    /*
+      Having reached the final time T, we want to measure the error that we have made.
+      This method is responsible for that. Saves the results in a ConvergenceTable object
+      which later we can print or compute things with it
+    */
+    void Post_Process();
+    /*
+      The whole reason for this class, and all this code after all, is to make convergence tests
+      with respect to the time discretization. For this reason we need to be able to vary the time step
+      we are going to work with. This method sets up the time step that is to be used in the given run
+    */
+    void set_dt( const double ddt );
+
+  protected:
+    // The type of method
+    Method_Formulation type;
+
+    // Discretization data
+    //// The polynomial degree
+    unsigned int deg;
+    //// The time step
+    double dt;
+
+    // Physical data:
+    //// initial time, final time, Reynolds number
+    double t_0, T, Re;
+    //// The external driving force
+    Force<dim> rhs;
+    //// The exact velocity (to apply boundary values)
+    Velocity<dim> vel_exact;
+    //// The boundary conditions
+    std::map<unsigned int, double> boundary_values;
+
+    // Finite Element Spaces data
+    //// The mesh
+    Triangulation<dim> triangulation;
+    //// The DoF handlers
+    DoFHandler<dim> dof_handler_velocity, dof_handler_pressure;
+    //// The polynomial spaces
+    FE_Q<dim> fe_velocity, fe_pressure;
+    //// The quadrature formulae
+    QGauss<dim> quadrature_pressure, quadrature_velocity;
+
+    /*
+      Linear Algebra Data
+        The sparsity patterns where the matrices will live
+    */
+    SparsityPattern spar_pattern_velocity, spar_pattern_pressure, spar_pattern_pres_vel;
+    /*
+      The actual matrices. The projection matrix never changes.
+      For the velocity a part of the matrix never changes (neither in time nor for each component),
+      namely the part related with the time derivative and the diffusion
+      but, the advection part changes and so we need dim+1 matrices. One to store
+      the constant part, and dim for each matrix at each iteration
+    */
+    SparseMatrix<double> vel_Laplace_plus_Mass, vel_it_matrix[dim], vel_Mass, vel_Laplace,
+                         pres_Laplace, pres_Mass, pres_Diff[dim];
+    /*
+      We need to regularize the Laplace operator for the pressure
+      for that we use a constraint
+    */
+    ConstraintMatrix pres_regularization;
+    //// The solutions at times n and (n-1)
+    Vector<double> pres_n, pres_n_minus_1, phi_n, phi_n_minus_1, u_n[dim], u_n_minus_1[dim],
+    //// The time extrapolation of the velocity, used to make the semi-implicit time discretization
+    //// of the advection term
+                  u_star[dim],
+    //// Right hand side for the component of the momentum equation
+                  force[dim],
+    //// Temporary arrays
+    v_tmp, pres_tmp;
+    //// The preconditioners
+//     SparseILU<double> prec_velocity[dim];
+    SparseDirectUMFPACK prec_velocity[dim], prec_mass, prec_pressure;
+
+    // The convergence table
+    ConvergenceTable convergence_table;
+
+    // Exception we will throw if the time step is invalid
+    DeclException2( ExcInvalidTimeStep, double, double, <<" The time step "<<arg1<<" is out of range."<<std::endl
+                                                                <<" The permitted range is (0,"<<arg2<<"]");
+
+    /*
+      A diffusion step. The boolean parameter that it takes indicates if we want the preconditioners
+      for each component of the velocity to be updated. Recall that the operator that these matrices
+      represent is of the form
+          1.5/dt Mass + 1/Re Laplace_h + ( u^*.D )
+      so the dominating part is
+          1.5/dt Mass + 1/Re Laplace_h
+      and we can recycle the preconditioner for several steps to save computational time
+    */
+    inline void diffusion_step( const bool reinit_prec );
+    /*
+        A projection step. The parameter that it takes indicates if we want the preconditioner to be
+        updated. Since this matrix never changes, ideally this should be done just once.
+    */
+    inline void projection_step( const bool reinit_prec );
+    /*
+      A pressure update step.
+      Capable of recognizing which formulation of the method we are using.
+      If it is the standard formulation, then the pressure update is done by
+          p^{n+1} = p^{n} + phi^{n+1}
+      which involves just adding numerical values.
+      On the other hand, for the rotational formulation of the method the pressure update
+      is done by the formula
+          p^{n+1} = p^{n} + phi^{n+1} +1/Re div u^{n+1}
+      which in any case involves the solution of a mass matrix problem.
+      NOTE: Here we assume that the mesh is uniformly refined, and thus we do not use
+      a preconditioner for this mass matrix problem
+    */
+    inline void update_pressure( const bool reinit_prec );
+  private:
+
+    // A bunch of data for the solvers and preconditioners
+    // data solve velocity
+    unsigned int vel_max_its, vel_Krylov_size, vel_off_diagonals, vel_update_prec;
+    double vel_eps, vel_diag_strength;
+    // data solve projection
+    unsigned int proj_max_its, proj_off_diagonals;
+    double proj_eps, proj_diag_strength;
+    // data solve pressure update
+    unsigned int pres_max_its, pres_off_diagonals;
+    double pres_eps, pres_diag_strength;
+
+    // Extremely simple function that computes the velocity extrapolant. Added for readability
+    inline void interpolate_velocity();
+
+    //////// initialization of constant wrt dt matrices
+    inline void init_velocity_matrices();
+    inline void init_pressure_matrices();
+    inline void init_gradient_operator();
+
+    // Assembly of the advection term
+    inline void assemble_advection_term( const unsigned int d );
+
+    // Solution of each component of the velocity on the diffusion step
+    inline void threaded_solve( const unsigned int d );
+    inline void plot_solution( const unsigned int step );
+};
+
+
+
+#endif
diff --git a/deal.II/examples/step-35/parameter-file.prm b/deal.II/examples/step-35/parameter-file.prm
new file mode 100644 (file)
index 0000000..52cc5ce
--- /dev/null
@@ -0,0 +1,61 @@
+# First a global definition
+# the type of method we want to use
+set Method_Form = rotational
+
+subsection Physical data
+  # In this subsection we declare the physical data
+  # The initial and final time, and the Reynolds number
+  set initial_time = 0.0
+  set final_time   = 1.0
+  set Reynolds     = 1.
+end
+
+subsection Time step data
+  # In this subsection we declare the data that is to be used for the convergence tests
+  # To test convergence in time we run the algorithm with the time step set from
+  # 'initial_dt' to 'final_dt', each time dividing the current time step by 'dt_decrement'
+  set initial_dt   = .1
+  set final_dt     = 5e-3
+  set dt_decrement = 2.
+end
+
+subsection Space discretization
+  # In this subsection we declare the data that is relevant to the space discretization
+  # we set the number of global refines the triangulation must have
+  # and the degree k of the pair Q_(k+1)--Q_k of velocity--pressure finite element spaces
+  set n_of_refines = 4
+  set pressure_fe_degree = 1
+end
+
+subsection Data solve velocity
+  # In this section we declare the parameters that are going to control the solution process
+  # for the velocity.
+  set max_iterations = 1000  # maximal number of iterations that GMRES must make
+  set eps            = 1e-12 # stopping criterion
+  set Krylov_size    = 10    # size of the Krylov subspace to be used in GMRES
+  set off_diagonals  = 60    # number of off diagonals that ILU must compute
+  set diag_strength  = 0     # diagonal strengthening value
+  set update_prec    = 20    # this number indicates how often the preconditioner must be updated
+end
+
+subsection Data solve projection
+  # In this section we declare the parameters that are going to control the solution process
+  # for the projection step.
+  set max_iterations = 1000  # maximal number of iterations that CG must make
+  set eps            = 1e-12 # stopping criterion
+  set off_diagonals  = 100    # number of off diagonals that ILU must compute. This one can be big as it is computed only once
+  set diag_strength  = 1.5     # diagonal strengthening value
+end
+
+subsection Data solve pressure update
+  # In this section we declare the parameters that are going to control the solution process
+  # for the pressure update step in the case of a rotational form of the method
+  set max_iterations = 1000  # maximal number of iterations that CG must make
+  set eps            = 1e-12 # stopping criterion
+  set off_diagonals  = 100    # number of off diagonals that ILU must compute. This one can be big as it is computed only once
+  set diag_strength  = 1e-5     # diagonal strengthening value
+end
+
+
+#Finally we set the verbosity level
+set verbose = false
\ No newline at end of file
diff --git a/deal.II/examples/step-35/source/EqData.cc b/deal.II/examples/step-35/source/EqData.cc
new file mode 100644 (file)
index 0000000..8a8e198
--- /dev/null
@@ -0,0 +1,170 @@
+/*
+    Implementation of the classes that represent the exact solution
+    and external force FOR THE PROBLEM ON THE DISK
+
+
+    There's not much to say here. All these classes just represent
+    mathematical formulae
+
+    by Abner Salgado.
+*/
+
+
+
+#include "../include/EqData.h"
+
+
+
+// Force function methods
+template<int dim> Force<dim>::Force( const double initial_time ): Multi_Component_Function<dim>( initial_time ){
+}
+
+
+
+template<int dim> void  Force<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                  const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Force<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline double Force<dim>::value(const Point<dim> &p, const unsigned int) const{
+  double t = FunctionTime::get_time(),
+         cosx = std::cos( p(0) ),
+         sinx = std::sin( p(0) ),
+         cosy = std::cos( p(1) ),
+         siny = std::sin( p(1) ),
+         cost = std::cos(t),
+         sint = std::sin(t),
+         return_value = 0.;
+  switch( Multi_Component_Function<dim>::component ){
+    case 0:
+    // y*sin(t)-x*cos(t)^2+cos(x)*sin(y)*sin(t)
+      return_value = p(1)*sint - p(0)*cost*cost + cosx*siny*sint;
+      break;
+    case 1:
+    // -x*sin(t)-y*cos(t)^2+sin(x)*cos(y)*sin(t)
+      return_value = -p(0)*sint - p(1)*cost*cost + sinx*cosy*sint ;
+
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+// Velocity function methods
+template<int dim> Velocity<dim>::Velocity(const double initial_time): Multi_Component_Function<dim>( initial_time ){
+}
+
+
+
+template<int dim> void  Velocity<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                  const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Velocity<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline double Velocity<dim>::value(const Point<dim> &p, const unsigned int) const{
+  double return_value = std::cos( Function<dim>::get_time() );
+  switch( Multi_Component_Function<dim>::component ){
+    case 0:
+     // -y*cos(t)
+       return_value *= -p(1);
+      break;
+    case 1:
+      // x*cos(t)
+      return_value *= p(0);
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+template<int dim> inline Tensor<1,dim> Velocity<dim>::gradient(const Point<dim> &p, const unsigned int) const{
+  Tensor<1,dim> return_value;
+  switch( Multi_Component_Function<dim>::component ){
+    // [0, -cos(t)]
+    case 0:
+     return_value[0] = 0.;
+     return_value[1] = -std::cos( Function<dim>::get_time() );
+      break;
+    case 1:
+    // [cos(t), 0]
+     return_value[0] = std::cos( Function<dim>::get_time() );
+     return_value[1] = 0.;
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+template<int dim> void Velocity<dim>::gradient_list( const std::vector<Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                                        const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    gradients[i] = Velocity<dim>::gradient( points[i] );
+}
+
+
+
+// Pressure function methods
+template<int dim> Pressure<dim>::Pressure(const double initial_time): Function<dim>(1,initial_time){}
+
+
+
+template<int dim> inline double Pressure<dim>::value(const Point<dim> &p, const unsigned int) const{
+  // sin(x-y+t)
+  return std::sin( p(0) )*std::sin( p(1) )*std::sin( Function<dim>::get_time() );
+}
+
+
+
+template<int dim> inline Tensor<1,dim> Pressure<dim>::gradient(const Point<dim> &p, const unsigned int) const{
+  // [cos(x)*sin(y)*sin(t), sin(x)*cos(y)*sin(t)]
+  return Point<dim>( std::cos( p(0) )*std::sin( p(1) )*std::sin( Function<dim>::get_time() ),
+                       std::sin( p(0) )*std::cos( p(1) )*std::sin( Function<dim>::get_time() ) );
+}
+
+
+
+template<int dim> void Pressure<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                    const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Pressure<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline void Pressure<dim>::gradient_list( const std::vector<Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                                                const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    gradients[i] = Pressure<dim>::gradient( points[i] );
+}
+
+
+
+// explicit template instantiation
+template class Force<deal_II_dimension>;
+template class Velocity<deal_II_dimension>;
+template class Pressure<deal_II_dimension>;
diff --git a/deal.II/examples/step-35/source/EqData_square.ccc b/deal.II/examples/step-35/source/EqData_square.ccc
new file mode 100644 (file)
index 0000000..2f52b5a
--- /dev/null
@@ -0,0 +1,211 @@
+/*
+    Implementation of the classes that represent the exact solution
+    and external force.
+
+    There's not much to say here. All these classes just represent
+    mathematical formulae
+
+    by Abner Salgado.
+*/
+
+
+
+#include "../include/EqData.h"
+
+
+
+// Force function methods
+template<int dim> Force<dim>::Force( const double initial_time ): Multi_Component_Function<dim>( initial_time ){
+}
+
+
+
+template<int dim> void  Force<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                  const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Force<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline double Force<dim>::value(const Point<dim> &p, const unsigned int) const{
+  double t = FunctionTime::get_time(),
+         sin_2pi_x  = std::sin( 2.*PI*p(0) ),
+         sin_2pi_y  = std::sin( 2.*PI*p(1) ),
+         cos_2pi_x  = std::cos( 2.*PI*p(0) ),
+         cos_2pi_y  = std::cos( 2.*PI*p(1) ),
+         sin_pi_x   = std::sin( PI*p(0) ),
+         sin_pi_y   = std::sin( PI*p(1) ),
+         cos_pi_x   = std::cos( PI*p(0) ),
+         cos_pi_y   = std::cos( PI*p(1) ),
+         cos_t      = std::cos( t ),
+         sin_t      = std::sin( t ),
+         return_value = 0.;
+  switch( Multi_Component_Function<dim>::component ){
+    case 0:
+    // Pi*sin(2*Pi*y)*sin(Pi*x)^2*cos(t)
+    // -2*Pi^3*sin(2*Pi*y)*cos(Pi*x)^2*sin(t)
+    // +6*Pi^3*sin(2*Pi*y)*sin(Pi*x)^2*sin(t)
+    // +2*Pi^3*sin(2*Pi*y)^2*sin(Pi*x)^3*sin(t)^2*cos(Pi*x)
+    // -2*Pi^3*sin(2*Pi*x)*sin(Pi*y)^2*sin(t)^2*cos(2*Pi*y)*sin(Pi*x)^2
+    // -Pi*sin(Pi*x)*sin(Pi*y)*sin(t)
+      return_value = PI*sin_2pi_y*sin_pi_x*sin_pi_x*cos_t
+                   - 2.*PI*PI*PI*sin_2pi_y*cos_pi_x*cos_pi_x*sin_t
+                   + 6.*PI*PI*PI*sin_2pi_y*sin_pi_x*sin_pi_x*sin_t
+                   + 2.*PI*PI*PI*sin_2pi_y*sin_2pi_y*sin_pi_x*sin_pi_x*sin_pi_x*sin_t*sin_t*cos_pi_x
+                   - 2.*PI*PI*PI*sin_2pi_x*sin_pi_y*sin_pi_y*sin_t*sin_t*cos_2pi_y*sin_pi_x*sin_pi_x
+                   - PI*sin_pi_x*sin_pi_y*sin_t;
+
+      break;
+    case 1:
+    // -Pi*sin(2*Pi*x)*sin(Pi*y)^2*cos(t)
+    // -6*Pi^3*sin(2*Pi*x)*sin(Pi*y)^2*sin(t)
+    // +2*Pi^3*sin(2*Pi*x)*cos(Pi*y)^2*sin(t)
+    // +Pi*cos(Pi*x)*cos(Pi*y)*sin(t)
+    // -2*Pi^3*sin(2*Pi*y)*sin(Pi*x)^2*sin(t)^2*cos(2*Pi*x)*sin(Pi*y)^2
+    // +2*Pi^3*sin(2*Pi*x)^2*sin(Pi*y)^3*sin(t)^2*cos(Pi*y)
+      return_value = -PI*sin_2pi_x*sin_pi_y*sin_pi_y*cos_t
+                   - 6.*PI*PI*PI*sin_2pi_x*sin_pi_y*sin_pi_y*sin_t
+                   + 2.*PI*PI*PI*sin_2pi_x*cos_pi_y*cos_pi_y*sin_t
+                   + PI*cos_pi_x*cos_pi_y*sin_t
+                   - 2.*PI*PI*PI*sin_2pi_y*sin_pi_x*sin_pi_x*sin_t*sin_t*cos_2pi_x*sin_pi_y*sin_pi_y
+                   + 2.*PI*PI*PI*sin_2pi_x*sin_2pi_x*sin_pi_y*sin_pi_y*sin_pi_y*sin_t*sin_t*cos_pi_y;
+
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+// Velocity function methods
+template<int dim> Velocity<dim>::Velocity(const double initial_time): Multi_Component_Function<dim>( initial_time ){
+}
+
+
+
+template<int dim> void  Velocity<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                  const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Velocity<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline double Velocity<dim>::value(const Point<dim> &p, const unsigned int) const{
+  double sin_2pi_x  = std::sin( 2.*PI*p(0) ),
+         sin_2pi_y  = std::sin( 2.*PI*p(1) ),
+         sin_pi_x   = std::sin( PI*p(0) ),
+         sin_pi_y   = std::sin( PI*p(1) ),
+         sin_t      = std::sin( Function<dim>::get_time() ),
+         return_value = 0.;
+  switch( Multi_Component_Function<dim>::component ){
+    case 0:
+     // pi*sin(2*pi*y)*sin(pi*x)^2*sin(t)
+       return_value = PI*sin_2pi_y*sin_pi_x*sin_pi_x*sin_t;
+      break;
+    case 1:
+      // -pi*sin(2*pi*x)*sin(pi*y)^2*sin(t)
+      return_value = -PI*sin_2pi_x*sin_pi_y*sin_pi_y*sin_t;
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+template<int dim> inline Tensor<1,dim> Velocity<dim>::gradient(const Point<dim> &p, const unsigned int) const{
+  Tensor<1,dim> return_value;
+  double sin_2pi_x = std::sin( 2.*PI*p(0) ),
+         sin_2pi_y = std::sin( 2.*PI*p(1) ),
+         cos_2pi_x = std::cos( 2.*PI*p(0) ),
+         cos_2pi_y = std::cos( 2.*PI*p(1) ),
+         sin_pi_x  = std::sin( PI*p(0) ),
+         sin_pi_y  = std::sin( PI*p(1) ),
+         cos_pi_x  = std::cos( PI*p(0) ),
+         cos_pi_y  = std::cos( PI*p(1) ),
+         sin_t     = std::sin( Function<dim>::get_time() );
+  switch( Multi_Component_Function<dim>::component ){
+    // [2*Pi^2*sin(2*Pi*y)*sin(Pi*x)*sin(t)*cos(Pi*x),
+    //     2*Pi^2*cos(2*Pi*y)*sin(Pi*x)^2*sin(t)]
+    case 0:
+     return_value[0] = 2.*PI*PI*sin_2pi_y*sin_pi_x*sin_t*cos_pi_x;
+     return_value[1] = 2.*PI*PI*cos_2pi_y*sin_pi_x*sin_pi_x*sin_t;
+      break;
+    case 1:
+    // [-2*Pi^2*cos(2*Pi*x)*sin(Pi*y)^2*sin(t),
+    // -2*Pi^2*sin(2*Pi*x)*sin(Pi*y)*sin(t)*cos(Pi*y)]
+     return_value[0] = -2.*PI*PI*cos_2pi_x*sin_pi_y*sin_pi_y*sin_t;
+     return_value[1] = -2.*PI*PI*sin_2pi_x*sin_pi_y*sin_t*cos_pi_y;
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+  return return_value;
+}
+
+
+
+template<int dim> void Velocity<dim>::gradient_list( const std::vector<Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                                        const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    gradients[i] = Velocity<dim>::gradient( points[i] );
+}
+
+
+
+// Pressure function methods
+template<int dim> Pressure<dim>::Pressure(const double initial_time): Function<dim>(1,initial_time){}
+
+
+
+template<int dim> inline double Pressure<dim>::value(const Point<dim> &p, const unsigned int) const{
+  // cos(pi*x)*sin(pi*y)*sin(t)
+  return std::cos( PI*p(0) )*std::sin( PI*p(1) )*std::sin( Function<dim>::get_time() );
+}
+
+
+
+template<int dim> inline Tensor<1,dim> Pressure<dim>::gradient(const Point<dim> &p, const unsigned int) const{
+  // || -sin(Pi*x)*Pi*sin(Pi*y)*sin(t) ||
+  // || cos(Pi*x)*cos(Pi*y)*Pi*sin(t)  ||
+  return ( PI*std::sin( Function<dim>::get_time() ) )* Point<dim>( -std::sin( PI*p(0) )*std::sin( PI*p(1) ),
+                                                                    std::cos( PI*p(0) )*std::cos( PI*p(1) ) ) ;
+}
+
+
+
+template<int dim> void Pressure<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                    const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    values[i] = Pressure<dim>::value( points[i] );
+}
+
+
+
+template<int dim> inline void Pressure<dim>::gradient_list( const std::vector<Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                                                const unsigned int ) const{
+  const unsigned int n_points = points.size();
+  Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+  for (unsigned int i=0; i<n_points; ++i)
+    gradients[i] = Pressure<dim>::gradient( points[i] );
+}
+
+
+
+// explicit template instantiation
+template class Force<deal_II_dimension>;
+template class Velocity<deal_II_dimension>;
+template class Pressure<deal_II_dimension>;
diff --git a/deal.II/examples/step-35/source/FileReader.cc b/deal.II/examples/step-35/source/FileReader.cc
new file mode 100644 (file)
index 0000000..ce58a8c
--- /dev/null
@@ -0,0 +1,187 @@
+#include "../include/FileReader.h"
+
+
+
+// Constructor. Here we set up the
+// Data Format we are going to use
+Data_Storage::Data_Storage(){
+  prm.declare_entry( "Method_Form", "rotational", Patterns::Selection( "rotational|standard" ),
+                    " Used to select the type of method that we are going to use. " );
+
+  // Physical data of the problem
+  prm.enter_subsection( "Physical data" );
+    prm.declare_entry( "initial_time", "0.", Patterns::Double( 0. ), " The initial time of the simulation. " );
+    prm.declare_entry( "final_time", "1.", Patterns::Double( 0. ), " The final time of the simulation. " );
+    prm.declare_entry( "Reynolds", "1.", Patterns::Double( 0. ), " The Reynolds number. " );
+  prm.leave_subsection();
+
+  // Time stepping data of the problem
+  prm.enter_subsection( "Time step data" );
+    prm.declare_entry( "initial_dt", "0.1", Patterns::Double( 0. ), " The initial time step size. " );
+    prm.declare_entry( "final_dt", "5e-4",  Patterns::Double( 0. ), " The final time step size. " );
+    prm.declare_entry( "dt_decrement", "2.", Patterns::Double( 1.5 ), " The factor by which the time step will be divided. " );
+  prm.leave_subsection();
+
+  // Space discretization data
+  prm.enter_subsection( "Space discretization" );
+    prm.declare_entry( "n_of_refines", "5", Patterns::Integer( 1, 15), " The number of global refines we do on the mesh. " );
+    prm.declare_entry( "pressure_fe_degree", "1", Patterns::Integer( 1, 5 ), " The polynomial degree for the pressure space. " );
+  prm.leave_subsection();
+
+  // Velocity solution data
+  prm.enter_subsection( "Data solve velocity" );
+    prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ), " The maximal number of iterations GMRES must make. " );
+    prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+    prm.declare_entry( "Krylov_size", "30", Patterns::Integer(1), " The size of the Krylov subspace to be used. " );
+    prm.declare_entry( "off_diagonals", "60", Patterns::Integer(0), " The number of off-diagonal elements ILU must compute. " );
+    prm.declare_entry( "diag_strength", "0.01", Patterns::Double( 0. ), " Diagonal strengthening coefficient. " );
+    prm.declare_entry( "update_prec", "15", Patterns::Integer(1), " This number indicates how often we need to update the preconditioner" );
+  prm.leave_subsection();
+
+  // Projection step data
+  prm.enter_subsection( "Data solve projection" );
+    prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ), " The maximal number of iterations CG must make. " );
+    prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+    prm.declare_entry( "off_diagonals", "100", Patterns::Integer(1), " The number of off-diagonal elements ILU must compute" );
+    prm.declare_entry( "diag_strength", "0.1", Patterns::Double( 0. ), " Diagonal strengthening coefficient. " );
+  prm.leave_subsection();
+
+  // Pressure update data
+  prm.enter_subsection( "Data solve pressure update" );
+    prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ), " The maximal number of iterations CG must make. " );
+    prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+    prm.declare_entry( "off_diagonals", "10", Patterns::Integer(0), " The number of off-diagonal elements that ILU must compute" );
+    prm.declare_entry( "diag_strength", "0.", Patterns::Double(0), " Diagonal strengthening coefficient" );
+  prm.leave_subsection();
+
+  // Verbosity of output
+  prm.declare_entry( "verbose", "true", Patterns::Bool(), " This indicates whether the output of the solution process should be verbose. " );
+
+  // How often we want the data to be outputted
+  prm.declare_entry( "output", "10", Patterns::Integer(1), " This indicates between how many time steps we print the solution. " );
+}
+
+
+
+// Destructor. Does nothing
+Data_Storage::~Data_Storage(){
+}
+
+
+
+// Here is where all happens. We read all the data from the indicated file
+void Data_Storage::read_data( char *filename ){
+  std::ifstream file;
+  file.open( filename );
+  if( not file )
+    throw ExcFileNotOpen( filename );
+  prm.read_input( file );
+
+  std::string token = prm.get( "Method_Form" );
+  if( token == std::string("rotational") )
+    form = METHOD_ROTATIONAL;
+  else
+    form = METHOD_STANDARD;
+
+  // Physical data of the problem
+  prm.enter_subsection( "Physical data" );
+    initial_time = prm.get_double( "initial_time" );
+    final_time   = prm.get_double( "final_time" );
+    Reynolds     = prm.get_double( "Reynolds" );
+  prm.leave_subsection();
+
+  // Time stepping data of the problem
+  prm.enter_subsection( "Time step data" );
+    initial_dt   = prm.get_double( "initial_dt" );
+    final_dt     = prm.get_double( "final_dt" );
+    dt_decrement = prm.get_double( "dt_decrement" );
+  prm.leave_subsection();
+
+  // Space discretization data
+  prm.enter_subsection( "Space discretization" );
+    n_of_global_refines = prm.get_integer( "n_of_refines" );
+    pressure_degree     = prm.get_integer( "pressure_fe_degree" );
+  prm.leave_subsection();
+
+  // Velocity solution data
+  prm.enter_subsection( "Data solve velocity" );
+    vel_max_iterations = prm.get_double( "max_iterations" );
+    vel_eps            = prm.get_double( "eps" );
+    vel_Krylov_size    = prm.get_integer( "Krylov_size" );
+    vel_off_diagonals  = prm.get_integer( "off_diagonals" );
+    vel_diag_strength  = prm.get_double( "diag_strength" );
+    vel_update_prec    = prm.get_integer( "update_prec" );
+  prm.leave_subsection();
+
+  // Projection step data
+  prm.enter_subsection( "Data solve projection" );
+    proj_max_iterations = prm.get_integer( "max_iterations" );
+    proj_eps            = prm.get_double( "eps" );
+    proj_off_diagonals  = prm.get_integer( "off_diagonals" );
+    proj_diag_strength  = prm.get_double( "diag_strength" );
+  prm.leave_subsection();
+
+  // Pressure update data
+  prm.enter_subsection( "Data solve pressure update" );
+    pres_max_iterations = prm.get_integer( "max_iterations" );
+    pres_eps            = prm.get_double( "eps" );
+    pres_off_diagonals  = prm.get_integer( "off_diagonals" );
+    pres_diag_strength  = prm.get_double( "diag_strength" );
+  prm.leave_subsection();
+
+  // Verbosity
+  verbose = prm.get_bool( "verbose" );
+
+  // Output frequency
+  output = prm.get_integer( "output" );
+
+  file.close();
+}
+
+
+
+// Prints the current values of the data
+// Mostly (if not only) used for debugging purposes
+void Data_Storage::print_status() const{
+  std::cout<<"Method Form = "<<( (form == METHOD_ROTATIONAL)?"rotational":"standard" )<<std::endl
+           <<"Physical Data:"<<std::endl
+           <<"  initial_time = "<<initial_time<<std::endl
+           <<"  final_time = "<<final_time<<std::endl
+           <<"  Re = "<<Reynolds<<std::endl<<std::endl
+           <<"Time step data:"<<std::endl
+           <<"  initial_dt = "<<initial_dt<<std::endl
+           <<"  final_dt = "<<final_dt<<std::endl
+           <<"  dt_decrement = "<<dt_decrement<<std::endl<<std::endl
+           <<"Space discretization data:"<<std::endl
+           <<"  n_of_global_refines = "<<n_of_global_refines<<std::endl
+           <<"  Pressure space degree = "<<pressure_degree<<std::endl<<std::endl
+           <<"Data for the solution of the diffusion:"<<std::endl
+           <<"  max iterations = "<<vel_max_iterations<<std::endl
+           <<"  eps = "<<vel_eps<<std::endl
+           <<"  Krylov subspace size = "<<vel_Krylov_size<<std::endl
+           <<"  off diagonals = "<<vel_off_diagonals<<std::endl
+           <<"  diagonal strengthening = "<<vel_diag_strength<<std::endl
+           <<"  update prec every "<<vel_update_prec<<" steps"<<std::endl<<std::endl
+           <<"Data for the projection step:"<<std::endl
+           <<"  max iterations = "<<proj_max_iterations<<std::endl
+           <<"  eps = "<<proj_eps<<std::endl
+           <<"  off diagonals = "<<proj_off_diagonals<<std::endl
+           <<"  diagonal strengthening = "<<proj_diag_strength<<std::endl<<std::endl
+           <<"Data for the pressure update step"<<std::endl
+           <<"  max iterations = "<<pres_max_iterations<<std::endl
+           <<"  eps = "<<pres_eps<<std::endl
+           <<"  off_diagonals = "<<pres_off_diagonals<<std::endl
+           <<"  diagonal strengthening = "<<pres_diag_strength<<std::endl<<std::endl
+           <<"Verbose = "<<(verbose?"true":"false")<<std::endl<<std::endl
+           <<"Output ="<<output<<std::endl<<std::endl;
+}
+
+
+
+// This method just prints the format the
+// input data file should have
+void Data_Storage::print_usage(){
+  static const char *message = "";
+  std::cout<<message<<std::endl;
+  prm.print_parameters( std::cout, ParameterHandler::Text );
+}
diff --git a/deal.II/examples/step-35/source/MultiComponentFunction.cc b/deal.II/examples/step-35/source/MultiComponentFunction.cc
new file mode 100644 (file)
index 0000000..5624c9d
--- /dev/null
@@ -0,0 +1,26 @@
+/*
+    Muti_Component_Function class implementation
+*/
+#include "../include/MultiComponentFunction.h"
+
+
+// Constructor: Just call the Function constructor and set the component to 0
+template<int dim> Multi_Component_Function<dim>::Multi_Component_Function( const double initial_time ):
+                    Function<dim>( 1, initial_time ), component(0) {
+}
+
+
+
+// Set Component Function: Check that it is in range and then set it
+template<int dim> void Multi_Component_Function<dim>::set_component(const unsigned int d ){
+  // Check if the requested component is correct
+  Assert( d<dim, ExcIndexRange( d, 0, dim ) );
+  // We have only written 2d
+  Assert( dim >= 2, ExcNotImplemented() );
+  component = d;
+}
+
+
+
+// explicit template instantiation
+template class Multi_Component_Function<deal_II_dimension>;
diff --git a/deal.II/examples/step-35/source/NavierStokes.cc b/deal.II/examples/step-35/source/NavierStokes.cc
new file mode 100644 (file)
index 0000000..88961bd
--- /dev/null
@@ -0,0 +1,792 @@
+/*
+    Implementation of the Navier_Stokes_projection class
+    by Abner Salgado.
+*/
+#include "../include/NavierStokes.h"
+
+
+
+#include <sys/time.h>
+
+/// debug
+bool show;
+double Solve_time;
+///----
+
+
+// Constructor
+template<int dim> Navier_Stokes_Projection<dim>::Navier_Stokes_Projection( const Data_Storage &data):
+                    type( data.form ), deg( data.pressure_degree ), dt( data.initial_dt ), t_0( data.initial_time ),
+                    T( data.final_time ), Re( data.Reynolds ), rhs( data.initial_time ), vel_exact( data.initial_time ),
+                    dof_handler_velocity(triangulation), dof_handler_pressure(triangulation),
+                    fe_velocity(deg+1), fe_pressure(deg), quadrature_pressure(deg+1), quadrature_velocity(deg+2),
+                    vel_max_its( data.vel_max_iterations ), vel_Krylov_size( data.vel_Krylov_size ),
+                    vel_off_diagonals( data.vel_off_diagonals ),
+                    vel_update_prec( data.vel_update_prec ), vel_eps( data.vel_eps ), vel_diag_strength( data.vel_diag_strength),
+                    proj_max_its( data.proj_max_iterations ), proj_off_diagonals( data.proj_off_diagonals ),
+                    proj_eps( data.proj_eps ), proj_diag_strength( data.proj_diag_strength ),
+                    pres_max_its( data.pres_max_iterations), pres_off_diagonals( data.pres_off_diagonals ),
+                    pres_eps( data.pres_eps ), pres_diag_strength( data.pres_diag_strength )
+{
+  // After having initialized the bunch of data we do nothing
+  // NOTE TO SELF: Do I need to do this check?
+  if(deg < 1)
+  std::cout<<" WARNING: The chosen pair of finite element spaces is not stable."<<std::endl
+           <<" The obtained results will be nonsense"<<std::endl;
+  // We check that the time step is within the allowed limits
+  AssertThrow( not ( ( dt <= 0. ) or ( dt > .5*T ) ), ExcInvalidTimeStep( dt, .5*T ) );
+}
+
+
+
+// Destructor
+template<int dim> Navier_Stokes_Projection<dim>::~Navier_Stokes_Projection(){
+  dof_handler_velocity.clear();
+  dof_handler_pressure.clear();
+}
+
+
+
+// Set time step
+template<int dim> void Navier_Stokes_Projection<dim>::set_dt( const double ddt ){
+  // We just check that it is within the permitted limits
+  AssertThrow( not ( ( ddt <= 0. ) or ( ddt > .5*T ) ), ExcInvalidTimeStep( ddt, .5*T ) );
+  dt = ddt;
+}
+
+
+
+// Initialization of the velocity matrices and assembly of those that do not depend on dt
+template<int dim> void Navier_Stokes_Projection<dim>::init_velocity_matrices(){
+  //// Init the sparsity pattern for the velocity
+  spar_pattern_velocity.reinit( dof_handler_velocity.n_dofs(), dof_handler_velocity.n_dofs(),
+                                dof_handler_velocity.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_velocity, spar_pattern_velocity );
+  spar_pattern_velocity.compress();
+
+  //// Init the matrices for the velocity
+  vel_Laplace_plus_Mass.reinit( spar_pattern_velocity );
+  for( unsigned int d=0; d<dim; ++d )
+    vel_it_matrix[d].reinit( spar_pattern_velocity );
+  vel_Mass.reinit( spar_pattern_velocity );
+  vel_Laplace.reinit( spar_pattern_velocity );
+
+  // We assemble the mass matrix
+  MatrixCreator::create_mass_matrix( dof_handler_velocity, quadrature_velocity, vel_Mass );
+
+  // We assemble the Laplace matrix
+  MatrixCreator::create_laplace_matrix( dof_handler_velocity, quadrature_velocity, vel_Laplace );
+}
+
+
+
+template<int dim> void Navier_Stokes_Projection<dim>::init_pressure_matrices(){
+  //// Init the sparsity pattern for the pressure
+  spar_pattern_pressure.reinit( dof_handler_pressure.n_dofs(), dof_handler_pressure.n_dofs(),
+                                dof_handler_pressure.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_pressure, spar_pattern_pressure );
+
+  // Before we close the sparsity pattern, we need to
+  // init the constraints for the Laplace operator on the pressure space
+  pres_regularization.clear();
+//  DoFTools::make_hanging_node_constraints( dof_handler_pressure, pres_regularization ); //??
+
+  // Add the only constraint that we have
+  pres_regularization.add_line(0);
+
+  // close it
+  pres_regularization.close();
+
+  // condense the sparsity pattern
+  pres_regularization.condense( spar_pattern_pressure );
+
+  // Compress the sparsity pattern for the pressure
+  spar_pattern_pressure.compress();
+
+  //// Init the matrices for the pressure
+  pres_Laplace.reinit( spar_pattern_pressure );
+  pres_Mass.reinit( spar_pattern_pressure );
+
+  // Now we assemble the matrices
+  // The Laplace operator is the projection matrix
+  MatrixCreator::create_laplace_matrix( dof_handler_pressure, quadrature_pressure, pres_Laplace );
+  // The pressure mass matrix to do the pressure update
+  MatrixCreator::create_mass_matrix(  dof_handler_pressure, quadrature_pressure, pres_Mass );
+
+  // Finally we condense the Laplace operator
+  pres_regularization.condense( pres_Laplace );
+}
+
+
+
+template<int dim> void Navier_Stokes_Projection<dim>::init_gradient_operator(){
+  //// Init the sparsity pattern for the gradient operator
+  spar_pattern_pres_vel.reinit( dof_handler_velocity.n_dofs(), dof_handler_pressure.n_dofs(),
+                                dof_handler_velocity.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_velocity, dof_handler_pressure, spar_pattern_pres_vel );
+  spar_pattern_pres_vel.compress();
+
+  /*
+      To assemble each component of the gradient operator
+      we need to make a loop over all cells and compute the products of
+          velocity * d_#(pressure)
+      where # is the current space component. For this reason we need two cell
+      iterators, one for the velocity and one for the pressure space.
+  */
+  typename DoFHandler<dim>::active_cell_iterator cell_init       = dof_handler_velocity.begin_active(),
+                                                   cell_end        = dof_handler_velocity.end(),
+                                                   cell,
+                                                   bogus_cell_init = dof_handler_pressure.begin_active(),
+                                                   bogus_cell;
+  /*
+    The FEValues extractors.
+    For the velocity we need values,
+    For the pressure we only need gradients.
+  */
+  FEValues<dim> fe_values_velocity( fe_velocity, quadrature_velocity, update_values | update_JxW_values ),
+                fe_values_pressure( fe_pressure, quadrature_velocity, update_gradients );
+
+  // Usual, and useful, abreviations
+  const unsigned int vel_dofs_per_cell  = fe_velocity.dofs_per_cell,
+                     pres_dofs_per_cell = fe_pressure.dofs_per_cell,
+                     n_q_points         = quadrature_velocity.size();
+
+  // The local gradient operator
+  FullMatrix<double> local_grad( vel_dofs_per_cell, pres_dofs_per_cell );
+
+  // Local to global DoF's map
+  std::vector<unsigned int> vel_local_dof_indices( vel_dofs_per_cell ), pres_local_dof_indices( pres_dofs_per_cell );
+
+  for( unsigned int d=0; d<dim; ++d ){
+    // we reinit the given component of the gradient operator
+    pres_Diff[d].reinit( spar_pattern_pres_vel );
+
+    // The usual cell loop
+    for( cell = cell_init, bogus_cell = bogus_cell_init; cell not_eq cell_end; ++cell, ++bogus_cell ){
+      fe_values_velocity.reinit( cell );
+      fe_values_pressure.reinit( bogus_cell );
+
+      cell->get_dof_indices( vel_local_dof_indices );
+      bogus_cell->get_dof_indices( pres_local_dof_indices );
+
+      // local contributions
+      local_grad = 0.;
+      for( unsigned int q=0; q<n_q_points; ++q ){
+        for( unsigned int i=0; i<vel_dofs_per_cell; ++i )
+          for( unsigned int j=0; j<pres_dofs_per_cell; ++j )
+            local_grad( i, j ) += fe_values_velocity.JxW(q)*fe_values_velocity.shape_value( i, q )
+                                  *fe_values_pressure.shape_grad( j, q)[d];
+      }
+
+      // Add it to the global contributions
+      for( unsigned int i=0; i<vel_dofs_per_cell; ++i )
+        for( unsigned int j=0; j<pres_dofs_per_cell; ++j)
+          pres_Diff[d].add( vel_local_dof_indices[i], pres_local_dof_indices[j], local_grad( i, j)  );
+    }
+  }
+}
+
+
+
+// Create Triangulation method
+template<int dim> void Navier_Stokes_Projection<dim>::Create_Triangulation( const unsigned int n_of_refines ){
+  // A disk
+  GridGenerator::hyper_ball( triangulation );
+  static const HyperBallBoundary<dim> boundary;
+  triangulation.set_boundary( 0, boundary );
+
+/*  // Our domain is a unit square.
+  GridGenerator::hyper_cube(triangulation);*/
+  triangulation.refine_global( n_of_refines );
+  std::cout<<" Number of active cells: "<<triangulation.n_active_cells()<<std::endl;
+
+  // Having created the mesh, we create the mesh-dependent data
+
+  //// Distribute the degrees of freedom and renumber them
+  dof_handler_velocity.distribute_dofs( fe_velocity );
+  DoFRenumbering::boost::Cuthill_McKee( dof_handler_velocity );
+  dof_handler_pressure.distribute_dofs( fe_pressure );
+  DoFRenumbering::boost::Cuthill_McKee( dof_handler_pressure );
+
+  // Initialize the matrices for the velocity
+  init_velocity_matrices();
+
+  // Initialize the matrices for the pressure
+  init_pressure_matrices();
+
+  // Initialize the gradient operator
+  init_gradient_operator();
+
+
+  //// Set the correct size for the various vectors that we need
+  pres_n.reinit( dof_handler_pressure.n_dofs() );
+  pres_n_minus_1.reinit( dof_handler_pressure.n_dofs() );
+
+  phi_n.reinit( dof_handler_pressure.n_dofs() );
+  phi_n_minus_1.reinit( dof_handler_pressure.n_dofs() );
+
+  pres_tmp.reinit( dof_handler_pressure.n_dofs() );
+
+  for(unsigned int d=0; d<dim; ++d){
+    u_n[d].reinit( dof_handler_velocity.n_dofs() );
+    u_n_minus_1[d].reinit( dof_handler_velocity.n_dofs() );
+    u_star[d].reinit( dof_handler_velocity.n_dofs() );
+    force[d].reinit( dof_handler_velocity.n_dofs() );
+  }
+  v_tmp.reinit( dof_handler_velocity.n_dofs() );
+
+  std::cout<<" dim( X_h ) = "<<( dof_handler_velocity.n_dofs()*dim )<<std::endl
+           <<" dim( M_h ) = "<<dof_handler_pressure.n_dofs()<<std::endl;
+}
+
+
+
+// Initialization of the time dependent data
+template<int dim> void Navier_Stokes_Projection<dim>::Initialize(){
+
+  // The constant part of the matrix for the velocity is 1.5/dt*Mass + 1/Re*Delta_h
+  vel_Laplace_plus_Mass = 0.;
+  vel_Laplace_plus_Mass.add( 1./Re, vel_Laplace );
+  vel_Laplace_plus_Mass.add( 1.5/dt, vel_Mass );
+
+  // Now we start the vectors and load them with the initial data
+  ////pressure
+  Pressure<dim> pres( t_0 );
+  VectorTools::interpolate( dof_handler_pressure, pres, pres_n_minus_1 );
+  pres.advance_time( dt );
+  VectorTools::interpolate( dof_handler_pressure, pres, pres_n );
+
+  ////phi
+  phi_n = 0.;
+  phi_n_minus_1 = 0.;
+
+  ////velocity
+  Velocity<dim> v;
+  for(unsigned int d=0; d<dim; ++d){
+    v.set_time( t_0 );
+    v.set_component(d);
+
+    VectorTools::interpolate( dof_handler_velocity, v, u_n_minus_1[d] );
+
+    v.advance_time( dt );
+    VectorTools::interpolate( dof_handler_velocity, v, u_n[d] );
+  }
+  plot_solution(1);
+}
+
+
+
+/*
+    A time marching step, the output messages and name of the functions are self
+    explanatory, so no comments here
+*/
+template<int dim> void Navier_Stokes_Projection<dim>::run( const bool verbose, const unsigned int n_of_plots ){
+  // We set the number of steps
+  unsigned int n_steps = ( T - t_0 )/dt;
+  // Set the source term to the correct time
+  rhs.set_time( 2.*dt );
+  vel_exact.set_time( 2.*dt );
+  // do as many steps as necessary
+  timeval init_time, end_time;
+
+  /// debug
+  show = false;
+  Solve_time = 0.;
+  ///-----
+
+  gettimeofday( &init_time, 0 );
+  for( unsigned int n = 2; n<=n_steps; ++n ){
+
+    /// debug
+    if( n == n_steps )
+      show = true;
+    ///----
+
+    if( verbose )
+      std::cout<<" Step = "<<n<<" Time = "<<(n*dt)<<std::endl;
+    if( verbose )
+      std::cout<<"  Interpolating the velocity "<<std::endl;
+    interpolate_velocity();
+    if( verbose )
+      std::cout<<"  Diffusion Step"<<std::endl;
+    if( ( n%vel_update_prec==0 ) and ( verbose ) )
+      std::cout<<"   With reinitialization of the preconditioner"<<std::endl;
+    diffusion_step( (n%vel_update_prec == 0 ) or ( n == 2) );
+    if( verbose )
+      std::cout<<"  Projection Step"<<std::endl;
+    projection_step( ( n == 2 ) );
+    if( verbose )
+      std::cout<<"  Updating the Pressure"<<std::endl;
+    update_pressure( ( n == 2 ) );
+    if( n%n_of_plots ==0 ){
+      if( verbose )
+        std::cout<<" Plotting Solution"<<std::endl;
+      plot_solution(n);
+    }
+    //  advance in time
+    rhs.advance_time(dt);
+    vel_exact.advance_time(dt);
+  }
+  gettimeofday( &end_time, 0 );
+  double run_time = double( end_time.tv_sec - init_time.tv_sec ) + 1e-6*double( end_time.tv_usec - init_time.tv_usec );
+  run_time /= n_steps;
+  run_time /= double( dim*dof_handler_velocity.n_dofs() + dof_handler_pressure.n_dofs() );
+//   if( verbose )
+    std::cout<<" Run time = "<<run_time<<std::endl;
+}
+
+
+
+// Simple procedure that interpolates the velocity
+template<int dim> void Navier_Stokes_Projection<dim>::interpolate_velocity(){
+  for( unsigned int d=0; d<dim; ++d ){
+    u_star[d] = 0.;
+    u_star[d].add( 2., u_n[d], -1., u_n_minus_1[d] );
+  }
+}
+
+
+
+template<int dim> void Navier_Stokes_Projection<dim>::assemble_advection_term( const unsigned int d ){
+  // Cell iterator
+  typename DoFHandler<dim>::active_cell_iterator cell     = dof_handler_velocity.begin_active(),
+                                                   cell_end = dof_handler_velocity.end();
+
+  /*
+    The FEValues extractors.
+    We need values because of the terms involving u^*
+    also we need the gradient to assemble the advection part
+  */
+  FEValues<dim> fe_values( fe_velocity, quadrature_velocity, update_values | update_JxW_values | update_gradients );
+
+  // Usual, and useful, abreviations
+  const unsigned int dofs_per_cell = fe_velocity.dofs_per_cell,
+                     n_q_points    = quadrature_velocity.size();
+
+  /*
+    We need to compute the values of several finite element functions (and some gradients)
+    at quadrature points. For this we use all these arrays.
+    The names are self-explanatory
+  */
+  std::vector<double> u_old_local( n_q_points );
+  std::vector< Point<dim> > u_star_local( n_q_points );
+  std::vector< Tensor<1,dim> > grad_u_star(n_q_points);
+
+  // Local to global DoF's map
+  std::vector<unsigned int> local_dof_indices( dofs_per_cell );
+
+  // Local contribution of the advection operator
+  Vector<double> local_rhs( dofs_per_cell );
+
+  // loop over cells
+  for( ; cell not_eq cell_end; ++cell ){
+    // reinit the stuff
+    fe_values.reinit( cell );
+    cell->get_dof_indices( local_dof_indices );
+
+    // create the local u^*
+    for( unsigned int s=0; s<dim; ++s ){
+      fe_values.get_function_values( u_star[s], u_old_local );
+      for( unsigned int q=0; q<n_q_points; ++q )
+        u_star_local[q](s) = u_old_local[q];
+    }
+
+    fe_values.get_function_gradients( u_star[d], grad_u_star );
+
+    local_rhs = 0.;
+    for( unsigned int q =0; q<n_q_points; ++q )
+      for( unsigned int i=0; i<dofs_per_cell; ++i )
+        // The local contribution of the advection term
+        local_rhs(i) -= ( ( u_star_local[q]*grad_u_star[q] )*fe_values.shape_value( i, q ) )*fe_values.JxW(q) ;
+
+    // Add local contributions to the global matrix
+    for( unsigned int i=0; i<dofs_per_cell; ++i )
+      force[d]( local_dof_indices[i] ) += local_rhs(i);
+  }
+}
+
+
+
+
+template<int dim> void Navier_Stokes_Projection<dim>::threaded_solve( const unsigned int d){
+
+  // Create the solver that we are going to use
+  SolverControl solver_control( vel_max_its, vel_eps*force[d].l2_norm() );
+  SolverCG<> cg( solver_control );
+  // Solve
+  cg.solve( vel_it_matrix[d], u_n[d], force[d], prec_velocity[d] );
+}
+
+
+
+// A diffusion step
+template<int dim> void Navier_Stokes_Projection<dim>::diffusion_step( const bool reinit_prec ){
+  // Used to solve in parallel the components of the velocity
+  Threads::ThreadGroup<> threads;
+
+  /// debug
+  static double assembly_time = 0.;
+  timeval init_time_assembly, end_time_assembly, init_time_solve, end_time_solve;
+  gettimeofday( &init_time_assembly, 0 );
+  ///----
+
+  /*
+    Define the pressure interpolant. It is defined by
+        p^* = p^{n} + 4/3 phi^{n} - 1/3 phi^{n-1}
+  */
+  pres_tmp = pres_n;
+  pres_tmp.add(4./3., phi_n, -1./3., phi_n_minus_1);
+  pres_tmp *= -1.;
+
+  for( unsigned int d=0; d<dim; ++d ){
+    // First we need to construct the right hand sides
+    rhs.set_component(d);
+    //// compute the source term
+    VectorTools::create_right_hand_side( dof_handler_velocity, quadrature_velocity, rhs, force[d] );
+
+    // We assemble the advection term
+    threads += Threads::spawn( *this, &Navier_Stokes_Projection<dim>::assemble_advection_term )(d);
+    //assemble_advection_term(d);
+
+    /*
+      Compute the value of the contribution of the old velocities.
+      Recall that BDF2 reads
+      ( 0.5/dt)*( 3u^{n+1} - 4u^{n} + u^{n-1} )
+      the terms involving u^{n} and u^{n-1} are known, so they go into the
+      right hand side
+    */
+    v_tmp = 0.;
+    v_tmp.add( 2./dt,u_n[d],-.5/dt,u_n_minus_1[d] );
+    vel_Mass.vmult_add( force[d], v_tmp );
+
+    // And add the contribution of the pressure part
+    pres_Diff[d].vmult_add( force[d], pres_tmp );
+
+    // Copy the old value
+    u_n_minus_1[d] = u_n[d];
+
+
+    // Create the matrix
+    vel_it_matrix[d].copy_from( vel_Laplace_plus_Mass );
+    //// Create the boundary values
+    vel_exact.set_component(d);
+    VectorTools::interpolate_boundary_values( dof_handler_velocity, 0, vel_exact, boundary_values );
+
+    // apply BCs
+    MatrixTools::apply_boundary_values( boundary_values, vel_it_matrix[d], u_n[d], force[d] );
+  }
+  threads.join_all();
+
+  ///  debug
+  gettimeofday( &end_time_assembly, 0 );
+  assembly_time += double( end_time_assembly.tv_sec - init_time_assembly.tv_sec )
+                            + 1e-6*double( end_time_assembly.tv_usec - init_time_assembly.tv_usec );
+  gettimeofday( &init_time_solve, 0 );
+  ///-----
+
+  for(unsigned int d=0; d<dim; ++d ){
+    // Check if the preconditioner needs reinitialization
+  //  if( reinit_prec )
+      prec_velocity[d].initialize( vel_it_matrix[d]/*, SparseILU<double>::AdditionalData( vel_diag_strength, vel_off_diagonals )*/ );
+
+    //prec_velocity[d].factorize( vel_it_matrix[d] );
+//    threaded_solve(d);
+
+//     And solve
+//    threads += Threads::spawn( *this, &Navier_Stokes_Projection<dim>::threaded_solve )(d);
+
+//     prec_velocity[d].solve( force[d] );
+//     u_n[d] = force[d];
+    prec_velocity[d].vmult( force[d], u_n[d] );
+  }
+//   threads.join_all();
+
+
+  ///debug
+  gettimeofday( &end_time_solve, 0 );
+  Solve_time += double( end_time_solve.tv_sec - init_time_solve.tv_sec )
+                            + 1e-6*double( end_time_solve.tv_usec - init_time_solve.tv_usec );
+  if(show)
+    std::cout<<" Assembly time = "<<assembly_time<<std::endl
+             <<" Solve time = "<<Solve_time<<std::endl;
+  ///----
+}
+
+
+
+// A projection step
+template<int dim> void Navier_Stokes_Projection<dim>::projection_step( const bool reinit_prec ){
+
+  /// debug
+  timeval init_time_assemble, init_time_solve, end_time_assemble, end_time_solve, init_time_factorize, end_time_factorize;
+  static double assemble_time=0., solve_time=0., factorize_time = 0.;
+  ///----
+
+
+  // Check if the preconditioner needs reinitialization
+  if( reinit_prec ){
+
+    /// debug
+    gettimeofday( &init_time_factorize, 0 );
+    ///-----
+// prec_pressure.initialize( pres_Laplace, SparseILU<double>::AdditionalData( proj_diag_strength, proj_off_diagonals ) );
+    prec_pressure.initialize( pres_Laplace );
+
+    /// debug
+    gettimeofday( &end_time_factorize, 0 );
+    factorize_time += double( end_time_factorize.tv_sec - init_time_factorize.tv_sec )
+                  + 1e-6*double( end_time_factorize.tv_usec - init_time_factorize.tv_usec );
+    ///--------
+  }
+
+  /// debug
+  gettimeofday( &init_time_assemble, 0 );
+  ///-----
+
+  // To make things faster we assembled the discrete gradient operator
+  // so we just need to multiply by its transpose
+  pres_tmp = 0.;
+  for( unsigned d=0; d<dim; ++d )
+    pres_Diff[d].Tvmult_add( pres_tmp, u_n[d] );
+
+  /// debug
+  gettimeofday( &end_time_assemble, 0 );
+  assemble_time += double( end_time_assemble.tv_sec - init_time_assemble.tv_sec )
+                 + 1e-6*double( end_time_assemble.tv_usec - init_time_assemble.tv_usec );
+  ///-----
+
+  // Up to this point, pres_tmp has the term < u_n, grad q >
+  // we will assemble this only once, so we
+  // Copy the old value of phi
+  phi_n_minus_1 = phi_n;
+
+  phi_n = pres_tmp;
+  phi_n *= 1.5/dt;
+
+  /// debug
+  gettimeofday( &init_time_solve, 0 );
+  ///-------
+
+  // condense because of the constraint
+  pres_regularization.condense( phi_n );
+
+
+  prec_pressure.solve( phi_n );
+  // The method that we are going to use
+//   SolverControl solver_control( proj_max_its, proj_eps*pres_tmp.l2_norm() );
+//   SolverCG<> cg( solver_control );
+
+
+  // And solve
+//   cg.solve( pres_Laplace, phi_n, pres_tmp, prec_pressure );
+
+  // After solving we need to distribute the constrained stuff
+  pres_regularization.distribute( phi_n );
+
+  /// debug
+  gettimeofday( &end_time_solve, 0 );
+  solve_time += double( end_time_solve.tv_sec - init_time_solve.tv_sec )
+                 + 1e-6*double( end_time_solve.tv_usec - init_time_solve.tv_usec );
+  if(show)
+    std::cout<<" DIV assembly time = "<<assemble_time<<std::endl
+             <<" LAP solve time = "<<solve_time<<std::endl
+             <<" LAP factorize time = "<<factorize_time<<std::endl;
+  ///------
+}
+
+
+
+// Pressure update step
+template<int dim> void Navier_Stokes_Projection<dim>::update_pressure( const bool reinit_prec ){
+
+  /// debug
+  static double solve_time = 0.;
+  timeval init_time_solve, end_time_solve;
+  ///---------
+
+  // Copy the old pressure
+  pres_n_minus_1 = pres_n;
+  switch( type ){
+    case METHOD_STANDARD:
+      /*
+        Simple standard case
+            p^{n+1} = p^n + \phi^{n+1}
+      */
+      pres_n += phi_n;
+      break;
+    case METHOD_ROTATIONAL:
+    {
+      /*
+        Rotational form
+            p^{n+1} = p^n + \phi^{n+1} - 1/Re div u^{n+1}
+      */
+
+      /// debug
+      gettimeofday( &init_time_solve, 0 );
+      ///------
+
+      // The preconditioner for the mass matrix problem
+      if( reinit_prec )
+        prec_mass.initialize( pres_Mass/*, SparseILU<double>::AdditionalData( pres_diag_strength, pres_off_diagonals )*/ );
+
+      // The solver for  the mass matrix
+//       SolverControl solver_control( pres_max_its, pres_eps*pres_tmp.l2_norm() );
+//       SolverCG<> cg( solver_control );
+//
+//       // Solve the mass matrix problem
+//       cg.solve( pres_Mass, pres_n, pres_tmp, prec_mass );
+
+      pres_n = pres_tmp;
+      prec_mass.solve( pres_n );
+
+      pres_n.sadd(1./Re, 1., pres_n_minus_1, 1., phi_n );
+
+      /// debug
+      gettimeofday( &end_time_solve, 0 );
+      solve_time += double( end_time_solve.tv_sec - init_time_solve.tv_sec )
+                 + 1e-6*double( end_time_solve.tv_usec - init_time_solve.tv_usec );
+      ///----------
+
+      break;
+    }
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+
+  /// debug
+  if(show)
+    std::cout<<" PRES_UPD solve time = "<<solve_time<<std::endl;
+  ///---
+
+}
+
+
+
+// Post Processing
+template<int dim> void Navier_Stokes_Projection<dim>::Post_Process(){
+  double tmp, vel_err_L2=0., vel_err_H1=0., pres_err_L2;
+
+  Vector<double> differences( triangulation.n_active_cells() );
+
+  Velocity<dim> u_exact(T);
+  for( unsigned int d=0; d<dim; ++d ){
+    u_exact.set_component(d);
+
+    // L2 error for this component of the velocity
+    differences = 0.;
+    VectorTools::integrate_difference( dof_handler_velocity, u_n[d], u_exact, differences, quadrature_velocity, VectorTools::L2_norm );
+    tmp = differences.l2_norm();
+    vel_err_L2 += tmp*tmp;
+
+    // H1 error for this component of the velocity
+    differences = 0.;
+    VectorTools::integrate_difference( dof_handler_velocity, u_n[d], u_exact, differences, quadrature_velocity, VectorTools::H1_seminorm );
+    tmp = differences.l2_norm();
+    vel_err_H1 += tmp*tmp;
+  }
+  vel_err_L2 = std::sqrt( vel_err_L2 );
+  vel_err_H1 = std::sqrt( vel_err_H1 );
+
+
+  /*
+    The actual pressure needs to have mean value zero.
+    We compute the mean value and subtract it from the computed pressure
+  */
+  double pres_mean_value = VectorTools::compute_mean_value( dof_handler_pressure, quadrature_pressure, pres_n, 0 );
+  for( unsigned int i=0; i<pres_n.size(); ++i )
+    pres_n(i) -= pres_mean_value;
+
+  Pressure<dim> pres_exact(T);
+
+  // L2 error of the pressure
+  differences = 0.;
+  VectorTools::integrate_difference( dof_handler_pressure, pres_n, pres_exact, differences, quadrature_pressure, VectorTools::L2_norm );
+  pres_err_L2 = differences.l2_norm();
+
+  // Add the obtained results to the convergence table
+  convergence_table.add_value( "dt"     , dt );
+  convergence_table.add_value( "u_L2"   , vel_err_L2 );
+  convergence_table.add_value( "u_H1"   , vel_err_H1 );
+  convergence_table.add_value( "pres_L2", pres_err_L2 );
+
+  convergence_table.set_precision( "dt"     , 5 );
+  convergence_table.set_precision( "u_L2"   , 5 );
+  convergence_table.set_precision( "u_H1"   , 5 );
+  convergence_table.set_precision( "pres_L2", 5 );
+
+  convergence_table.set_scientific( "u_L2"   , true );
+  convergence_table.set_scientific( "u_H1"   , true );
+  convergence_table.set_scientific( "pres_L2", true );
+
+  // And show it
+  convergence_table.write_text(std::cout);
+}
+
+
+
+// Output of the solution
+template<int dim> void Navier_Stokes_Projection<dim>::plot_solution( const unsigned int step ){
+  // This only works in 2d
+  Assert( dim==2, ExcNotImplemented() );
+
+  // We need to assemble the vorticy
+  FEValues<dim> fe_values( fe_velocity, quadrature_velocity, update_values | update_gradients | update_JxW_values );
+  const unsigned int dofs_per_cell = fe_velocity.n_dofs_per_cell(),
+                     n_q_points    = quadrature_velocity.size();
+  std::vector< Tensor<1,dim> > grad_vel_1( n_q_points ), grad_vel_2( n_q_points );
+  std::vector< unsigned int> local_dof_indices( dofs_per_cell );
+  Vector<double> local_rhs( dofs_per_cell );
+  double vorticity;
+
+  typename DoFHandler<dim>::active_cell_iterator cell = dof_handler_velocity.begin_active(),
+                                                   cend = dof_handler_velocity.end();
+
+  force[0] = 0.;
+  //We start the usual loop
+  for( ; cell not_eq cend; ++cell ){
+    // reinit all the needed stuff
+    local_rhs = 0.;
+    fe_values.reinit( cell );
+    cell->get_dof_indices( local_dof_indices );
+
+    // get the gradients of each function
+    fe_values.get_function_gradients( u_n[0], grad_vel_1 );
+    fe_values.get_function_gradients( u_n[1], grad_vel_2 );
+
+    // usual loop over quad points and local dofs
+    for( unsigned int q=0; q<n_q_points; ++q ){
+      vorticity = grad_vel_2[q][1] - grad_vel_1[q][0];
+      for( unsigned int i=0; i<dofs_per_cell; ++i )
+        local_rhs(i) += fe_values.JxW(q)*vorticity*fe_values.shape_value( i, q );
+    }
+
+    // And copy it to the local one
+    for( unsigned int i=0; i<dofs_per_cell; ++i )
+      force[0]( local_dof_indices[i] ) += local_rhs(i);
+  }
+
+  // We solve a mass matrix problem
+  SolverControl solver_control( 500, 1e-6*force[0].l2_norm() );
+  SolverCG<> cg( solver_control );
+  static bool is_prec_initted = false;
+  static SparseILU<double> prec;
+  if( not is_prec_initted ){
+    prec.initialize( vel_Mass, SparseILU<double>::AdditionalData( 1e-5, 70 ) );
+    is_prec_initted = true;
+  }
+  cg.solve( vel_Mass, force[1], force[0], prec );
+
+  // Once we have the vorticity we can output it
+  DataOut<dim> data_out;
+  data_out.attach_dof_handler( dof_handler_velocity );
+  data_out.add_data_vector( force[1], "vorticity" );
+  data_out.build_patches();
+  std::ostringstream filename;
+  filename<<"vorticity"<<step<<".vtk";
+  std::ofstream output( filename.str().c_str() );
+  data_out.write_vtk( output );
+}
+
+
+// explicit template instantiation
+template class Navier_Stokes_Projection<deal_II_dimension>;
diff --git a/deal.II/examples/step-35/source/main.cc b/deal.II/examples/step-35/source/main.cc
new file mode 100644 (file)
index 0000000..69d9738
--- /dev/null
@@ -0,0 +1,66 @@
+#include "../include/NavierStokes.h"
+
+
+#include <iostream>
+#include <sys/time.h>
+
+
+
+int main( int argc, char **argv ){
+  try{
+    Data_Storage data;
+    if( argc<2 ){
+      std::cout<<std::endl<<std::endl
+               <<" Missing parameter file!!"<<std::endl
+               <<" Aborting."<<std::endl<<std::endl;
+      data.print_usage();
+      exit(1);
+    }
+    data.read_data( argv[1] );
+    deallog.depth_console( data.verbose?2:0 );
+    Navier_Stokes_Projection<deal_II_dimension> test( data );
+    test.Create_Triangulation( data.n_of_global_refines );
+    timeval init_time, end_time;
+    gettimeofday( &init_time, 0 );
+    for( double dt = data.initial_dt; dt >= data.final_dt; dt /= data.dt_decrement ){
+      std::cout<<" dt = "<<dt<<std::endl;
+      test.set_dt( dt );
+      test.Initialize();
+      test.run( data.verbose, data.output );
+      test.Post_Process();
+      std::cout<<"====================================="<<std::endl<<std::endl;
+    }
+    gettimeofday( &end_time, 0 );
+    std::cout<<" Global time = "
+             <<( double( end_time.tv_sec - init_time.tv_sec )+1e-6*double( end_time.tv_usec - init_time.tv_usec ) )
+             <<" sec"<<std::endl;
+  }
+  catch (std::exception &exc){
+    std::cerr << std::endl << std::endl
+            << "----------------------------------------------------"
+            << std::endl;
+    std::cerr << "Exception on processing: " << std::endl
+            << exc.what() << std::endl
+            << "Aborting!" << std::endl
+            << "----------------------------------------------------"
+            << std::endl;
+    return 1;
+  }
+  catch (...){
+    std::cerr << std::endl << std::endl
+              << "----------------------------------------------------"
+              << std::endl;
+    std::cerr << "Unknown exception!" << std::endl
+              << "Aborting!" << std::endl
+              << "----------------------------------------------------"
+              << std::endl;
+    return 1;
+  }
+  std::cout<<"----------------------------------------------------"
+           <<std::endl
+           <<"Apparently everything went fine!"
+           <<std::endl
+           <<"Don't forget to brush your teeth :-)"
+           <<std::endl<<std::endl;
+  return 0;
+}
diff --git a/deal.II/examples/step-35/step-35.cc b/deal.II/examples/step-35/step-35.cc
new file mode 100644 (file)
index 0000000..d76bfbe
--- /dev/null
@@ -0,0 +1,1126 @@
+/* $Id: step-35.cc  $ */
+/* Version: $Name:  $ */
+/*      */
+/*    Copyright (C) 2007, 2008, 2009 by the deal.II authors */
+/*    Author: Abner Salgado, Texas A&M University 2009 */
+/*                                                                */
+/*    This file is subject to QPL and may not be  distributed     */
+/*    without copyright and license information. Please refer     */
+/*    to the file deal.II/doc/license.html for the  text  and     */
+/*    further information on this license.                        */
+
+
+// @sect3{Include Files}
+// We start by including all the necessary dealii header files and some C++ related ones. Each
+// one of them has been discussed in previous tutorial programs,
+// so we will not get into details here.
+#include <base/parameter_handler.h>
+#include <base/point.h>
+#include <base/function.h>
+#include <base/quadrature_lib.h>
+#include <base/convergence_table.h>
+#include <base/multithread_info.h>
+#include <base/thread_management.h>
+#include <base/work_stream.h>
+#include <base/parallel.h>
+
+#include <grid/tria.h>
+#include <grid/grid_generator.h>
+#include <grid/grid_refinement.h>
+#include <grid/tria_accessor.h>
+#include <grid/tria_iterator.h>
+#include <grid/tria_boundary_lib.h>
+#include <grid/grid_in.h>
+
+#include <dofs/dof_handler.h>
+#include <dofs/dof_accessor.h>
+#include <dofs/dof_tools.h>
+#include <dofs/dof_renumbering.h>
+#include <dofs/dof_constraints.h>
+
+#include <fe/fe_q.h>
+#include <fe/fe_values.h>
+#include <fe/fe_tools.h>
+#include <fe/fe_system.h>
+
+#include <lac/vector.h>
+#include <lac/sparse_matrix.h>
+#include <lac/solver_cg.h>
+#include <lac/precondition.h>
+#include <lac/solver_gmres.h>
+#include <lac/sparse_ilu.h>
+#include <lac/sparse_direct.h>
+
+#include <numerics/matrices.h>
+#include <numerics/vectors.h>
+#include <numerics/data_out.h>
+
+#include <fstream>
+#include <iostream>
+#include <cmath>
+#include <iostream>
+
+// Finally we import all the dealii names to the global namespace
+using namespace dealii;
+
+
+
+// @sect3{Run Time parameters}
+// Since our method has several options that can be fine-tuned we decided to group all these into
+// an external file, so that these can be determined at run-time.<br>
+// First, the formulation of the method, which we set as a <code>enum</code>.
+// Next, we declare a class that is going to read and store all the parameters that our program
+// needs to run.
+namespace RunTimeParameters{
+  enum Method_Formulation{
+    METHOD_STANDARD,
+    METHOD_ROTATIONAL
+  };
+
+  class Data_Storage{
+    public:
+      Data_Storage();
+      ~Data_Storage();
+      void read_data( const char *filename );
+      Method_Formulation form;
+      double initial_time,
+            final_time,
+            Reynolds;
+      double initial_dt,
+            final_dt,
+            dt_decrement;
+      unsigned int n_of_global_refines,
+                  pressure_degree;
+      unsigned int vel_max_iterations,
+                  vel_Krylov_size,
+                  vel_off_diagonals,
+                  vel_update_prec;
+      double vel_eps,
+            vel_diag_strength;
+      unsigned int proj_max_iterations,
+                  proj_off_diagonals;
+      double proj_eps,
+            proj_diag_strength;
+      unsigned int pres_max_iterations,
+                  pres_off_diagonals;
+      double pres_eps,
+            pres_diag_strength;
+      bool verbose;
+      unsigned int output;
+
+    protected:
+      ParameterHandler prm;
+  };
+
+// In the constructor of this class we declare all the parameters.
+// The details of how this works have been discussed somewhere else ***
+// so let's not elaborate on that
+
+  Data_Storage::Data_Storage(){
+    prm.declare_entry( "Method_Form", "rotational", Patterns::Selection( "rotational|standard" ),
+                      " Used to select the type of method that we are going to use. " );
+    prm.enter_subsection( "Physical data" );
+      prm.declare_entry( "initial_time", "0.", Patterns::Double( 0. ), " The initial time of the simulation. " );
+      prm.declare_entry( "final_time", "1.", Patterns::Double( 0. ), " The final time of the simulation. " );
+      prm.declare_entry( "Reynolds", "1.", Patterns::Double( 0. ), " The Reynolds number. " );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Time step data" );
+      prm.declare_entry( "initial_dt", "0.1", Patterns::Double( 0. ), " The initial time step size. " );
+      prm.declare_entry( "final_dt", "5e-4",  Patterns::Double( 0. ), " The final time step size. " );
+      prm.declare_entry( "dt_decrement", "2.", Patterns::Double( 1.5 ),
+                          " The factor by which the time step will be divided. " );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Space discretization" );
+      prm.declare_entry( "n_of_refines", "5", Patterns::Integer( 1, 15),
+                         " The number of global refines we do on the mesh. " );
+      prm.declare_entry( "pressure_fe_degree", "1", Patterns::Integer( 1, 5 ),
+                         " The polynomial degree for the pressure space. " );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve velocity" );
+      prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ),
+                         " The maximal number of iterations GMRES must make. " );
+      prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+      prm.declare_entry( "Krylov_size", "30", Patterns::Integer(1), " The size of the Krylov subspace to be used. " );
+      prm.declare_entry( "off_diagonals", "60", Patterns::Integer(0),
+                         " The number of off-diagonal elements ILU must compute. " );
+      prm.declare_entry( "diag_strength", "0.01", Patterns::Double( 0. ),
+                          " Diagonal strengthening coefficient. " );
+      prm.declare_entry( "update_prec", "15", Patterns::Integer(1),
+                         " This number indicates how often we need to update the preconditioner" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve projection" );
+      prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ),
+                         " The maximal number of iterations CG must make. " );
+      prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+      prm.declare_entry( "off_diagonals", "100", Patterns::Integer(1),
+                         " The number of off-diagonal elements ILU must compute" );
+      prm.declare_entry( "diag_strength", "0.1", Patterns::Double( 0. ), " Diagonal strengthening coefficient. " );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve pressure update" );
+      prm.declare_entry( "max_iterations", "1000", Patterns::Integer( 1, 1000 ),
+                         " The maximal number of iterations CG must make. " );
+      prm.declare_entry( "eps", "1e-12", Patterns::Double( 0. ), " The stopping criterion. " );
+      prm.declare_entry( "off_diagonals", "10", Patterns::Integer(0),
+                         " The number of off-diagonal elements that ILU must compute" );
+      prm.declare_entry( "diag_strength", "0.", Patterns::Double(0), " Diagonal strengthening coefficient" );
+    prm.leave_subsection();
+
+    prm.declare_entry( "verbose", "true", Patterns::Bool(),
+                       " This indicates whether the output of the solution process should be verbose. " );
+
+    prm.declare_entry( "output", "10", Patterns::Integer(1),
+                       " This indicates between how many time steps we print the solution. " );
+  }
+
+  Data_Storage::~Data_Storage(){}
+
+  void Data_Storage::read_data( const char *filename ){
+    std::ifstream file( filename );
+    if( not file )
+      throw ExcFileNotOpen( filename );
+    prm.read_input( file );
+
+    std::string token = prm.get( "Method_Form" );
+    if( token == std::string( "rotational" ) )
+      form = METHOD_ROTATIONAL;
+    else
+      form = METHOD_STANDARD;
+
+    prm.enter_subsection( "Physical data" );
+      initial_time = prm.get_double( "initial_time" );
+      final_time   = prm.get_double( "final_time" );
+      Reynolds     = prm.get_double( "Reynolds" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Time step data" );
+      initial_dt   = prm.get_double( "initial_dt" );
+      final_dt     = prm.get_double( "final_dt" );
+      dt_decrement = prm.get_double( "dt_decrement" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Space discretization" );
+      n_of_global_refines = prm.get_integer( "n_of_refines" );
+      pressure_degree     = prm.get_integer( "pressure_fe_degree" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve velocity" );
+      vel_max_iterations = prm.get_double( "max_iterations" );
+      vel_eps            = prm.get_double( "eps" );
+      vel_Krylov_size    = prm.get_integer( "Krylov_size" );
+      vel_off_diagonals  = prm.get_integer( "off_diagonals" );
+      vel_diag_strength  = prm.get_double( "diag_strength" );
+      vel_update_prec    = prm.get_integer( "update_prec" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve projection" );
+      proj_max_iterations = prm.get_integer( "max_iterations" );
+      proj_eps            = prm.get_double( "eps" );
+      proj_off_diagonals  = prm.get_integer( "off_diagonals" );
+      proj_diag_strength  = prm.get_double( "diag_strength" );
+    prm.leave_subsection();
+
+    prm.enter_subsection( "Data solve pressure update" );
+      pres_max_iterations = prm.get_integer( "max_iterations" );
+      pres_eps            = prm.get_double( "eps" );
+      pres_off_diagonals  = prm.get_integer( "off_diagonals" );
+      pres_diag_strength  = prm.get_double( "diag_strength" );
+    prm.leave_subsection();
+
+    verbose = prm.get_bool( "verbose" );
+
+    output = prm.get_integer( "output" );
+
+    file.close();
+  }
+}
+
+
+
+// @sect3{The Equation Data}
+// Here we declare the initial and boundary conditions, as well as the right hand side.
+namespace EquationData{
+  // Because of our implementation, we do not take advantage of the capabilities of the library
+  // to handle vector valued problems. Whether this is a good or bad idea is another issue. The
+  // point here is that we want to be able to write an interface for the equation data that is
+  // somehow dimension indenpendent. To be able to do that, our functions should be able to know
+  // on which space component we are currently working, and we should be able to have a
+  // common interface to do that. The following class is an attempt in that direction.
+  template<int dim> class MultiComponentFunction: public Function<dim>{
+    public:
+      MultiComponentFunction( const double initial_time = 0. );
+      void set_component( const unsigned int d );
+    protected:
+      unsigned int comp;
+  };
+
+  template<int dim> MultiComponentFunction<dim>::MultiComponentFunction( const double initial_time ):
+                                                  Function<dim>( 1, initial_time ), comp(0){}
+
+  template<int dim> void MultiComponentFunction<dim>::set_component(const unsigned int d ){
+    Assert( d<dim, ExcIndexRange( d, 0, dim ) );
+    comp = d;
+  }
+
+  template<int dim> class Velocity: public MultiComponentFunction<dim>{
+    public:
+      Velocity( const double initial_time = 0.0 );
+      virtual double value( const Point<dim> &p, const unsigned int component = 0 ) const;
+      virtual Tensor<1,dim> gradient( const Point<dim> &p, const unsigned int component = 0 ) const;
+      virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values,
+                                const unsigned int component = 0 ) const;
+      virtual void gradient_list( const std::vector< Point<dim> > &points,
+                                   std::vector< Tensor<1,dim> > &gradients,
+                                   const unsigned int component = 0 ) const;
+  };
+
+  template<int dim> Velocity<dim>::Velocity( const double initial_time ):
+                                    MultiComponentFunction<dim>( initial_time ){}
+
+  template<int dim> void  Velocity<dim>::value_list( const std::vector<Point<dim> > &points,
+                                                      std::vector<double> &values, const unsigned int ) const{
+    const unsigned int n_points = points.size();
+    Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+    for (unsigned int i=0; i<n_points; ++i)
+      values[i] = Velocity<dim>::value( points[i] );
+  }
+
+  template<int dim> inline double Velocity<dim>::value( const Point<dim> &p, const unsigned int ) const{
+    double return_value = std::cos( Function<dim>::get_time() );
+    switch( MultiComponentFunction<dim>::comp ){
+      case 0:
+        return_value *= -p(1);
+        break;
+      case 1:
+        return_value *= p(0);
+        break;
+      default:
+        Assert( false, ExcNotImplemented() );
+    };
+    return return_value;
+  }
+
+  template<int dim> inline Tensor<1,dim> Velocity<dim>::gradient( const Point<dim> &p, const unsigned int ) const{
+    Tensor<1,dim> return_value;
+    switch( MultiComponentFunction<dim>::comp ){
+      case 0:
+        return_value[0] = 0.;
+        return_value[1] = -std::cos( Function<dim>::get_time() );
+        break;
+      case 1:
+        return_value[0] = std::cos( Function<dim>::get_time() );
+        return_value[1] = 0.;
+        break;
+      default:
+        Assert( false, ExcNotImplemented() );
+    };
+    return return_value;
+  }
+
+  template<int dim> void Velocity<dim>::gradient_list( const std::vector<Point<dim> > &points,
+                                                         std::vector< Tensor<1,dim> > &gradients,
+                                                         const unsigned int ) const{
+    const unsigned int n_points = points.size();
+    Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+    for( unsigned int i=0; i<n_points; ++i )
+      gradients[i] = Velocity<dim>::gradient( points[i] );
+  }
+
+  template<int dim> class Pressure: public Function<dim>{
+    public:
+      Pressure( const double initial_time = 0.0 );
+      virtual double value( const Point<dim> &p, const unsigned int component = 0 ) const;
+      virtual Tensor<1,dim> gradient( const Point<dim> &p, const unsigned int component = 0 ) const;
+      virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values,
+                                const unsigned int component = 0 ) const;
+      virtual void gradient_list( const std::vector< Point<dim> > &points, std::vector< Tensor<1,dim> > &gradients,
+                                    const unsigned int component = 0 ) const;
+  };
+
+  template<int dim> Pressure<dim>::Pressure( const double initial_time ): Function<dim>( 1, initial_time ){}
+
+  template<int dim> inline double Pressure<dim>::value( const Point<dim> &p, const unsigned int ) const{
+    return std::sin( p(0) )*std::sin( p(1) )*std::sin( Function<dim>::get_time() );
+  }
+
+  template<int dim> inline Tensor<1,dim> Pressure<dim>::gradient( const Point<dim> &p, const unsigned int ) const{
+    return Point<dim>( std::cos( p(0) )*std::sin( p(1) )*std::sin( Function<dim>::get_time() ),
+                        std::sin( p(0) )*std::cos( p(1) )*std::sin( Function<dim>::get_time() ) );
+  }
+
+  template<int dim> void Pressure<dim>::value_list( const std::vector<Point<dim> > &points,
+                                                      std::vector<double> &values,
+                                                      const unsigned int ) const{
+    const unsigned int n_points = points.size();
+    Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+    for (unsigned int i=0; i<n_points; ++i)
+      values[i] = Pressure<dim>::value( points[i] );
+  }
+
+  template<int dim> inline void Pressure<dim>::gradient_list( const std::vector<Point<dim> > &points,
+                                                                 std::vector< Tensor<1,dim> > &gradients,
+                                                                 const unsigned int ) const{
+    const unsigned int n_points = points.size();
+    Assert( gradients.size() == n_points, ExcDimensionMismatch( gradients.size(), n_points ) );
+    for (unsigned int i=0; i<n_points; ++i)
+      gradients[i] = Pressure<dim>::gradient( points[i] );
+  }
+
+  template<int dim> class Force: public MultiComponentFunction<dim>{
+    public:
+      Force( const double initial_time =0.0 );
+      virtual double value( const Point<dim> &p, const unsigned int component = 0 ) const;
+      virtual void value_list( const std::vector< Point<dim> > &points, std::vector<double> &values,
+                                const unsigned int component = 0 ) const;
+  };
+
+  template<int dim> Force<dim>::Force( const double initial_time ):
+                                  MultiComponentFunction<dim>( initial_time ){}
+
+  template<int dim> void  Force<dim>::value_list( const std::vector<Point<dim> > &points, std::vector<double> &values,
+                                                    const unsigned int ) const{
+    const unsigned int n_points = points.size();
+    Assert( values.size() == n_points, ExcDimensionMismatch( values.size(), n_points ) );
+    for (unsigned int i=0; i<n_points; ++i)
+      values[i] = Force<dim>::value( points[i] );
+  }
+
+  template<int dim> inline double Force<dim>::value( const Point<dim> &p, const unsigned int ) const{
+    double t = Function<dim>::get_time(),
+          cosx = std::cos( p(0) ),
+          sinx = std::sin( p(0) ),
+          cosy = std::cos( p(1) ),
+          siny = std::sin( p(1) ),
+          cost = std::cos(t),
+          sint = std::sin(t),
+          return_value = 0.;
+    switch( MultiComponentFunction<dim>::comp ){
+      case 0:
+        return_value = p(1)*sint - p(0)*cost*cost + cosx*siny*sint;
+        break;
+      case 1:
+        return_value = -p(0)*sint - p(1)*cost*cost + sinx*cosy*sint ;
+
+        break;
+      default:
+        Assert( false, ExcNotImplemented() );
+    };
+    return return_value;
+  }
+}
+
+
+
+// @sect3{The <code>Navier_Stokes_Projection</code> class}
+// This is the main class of the program. It implements the various avatars of the projection
+// methods for Navier-Stokes equations.
+// The names for all the methods and attributes are self-explanatory.
+template<int dim> class Navier_Stokes_Projection{
+  public:
+    Navier_Stokes_Projection( const RunTimeParameters::Data_Storage &data );
+    ~Navier_Stokes_Projection();
+    void run( const bool verbose = false, const unsigned int n_of_plots = 10 );
+///
+    void Initialize();
+    void set_dt( const double ddt );
+    void Post_Process();
+///
+  protected:
+    RunTimeParameters::Method_Formulation type;
+
+    unsigned int deg;
+    double dt;
+
+    double t_0, T, Re;
+    EquationData::Force<dim> rhs;
+
+    EquationData::Velocity<dim> vel_exact;
+    std::map<unsigned int, double> boundary_values;
+
+    Triangulation<dim> triangulation;
+    DoFHandler<dim> dof_handler_velocity, dof_handler_pressure;
+    FE_Q<dim> fe_velocity, fe_pressure;
+    QGauss<dim> quadrature_pressure, quadrature_velocity;
+
+    SparsityPattern spar_pattern_velocity, spar_pattern_pressure, spar_pattern_pres_vel;
+    SparseMatrix<double> vel_Laplace_plus_Mass, vel_it_matrix[dim], vel_Mass, vel_Laplace,
+                         vel_Advection,
+                         pres_Laplace, pres_Mass, pres_Diff[dim];
+    ConstraintMatrix pres_regularization;
+    Vector<double> pres_n, pres_n_minus_1, phi_n, phi_n_minus_1, u_n[dim], u_n_minus_1[dim],
+                  u_star[dim],
+                  force[dim],
+                  v_tmp, pres_tmp;
+
+    SparseILU<double> prec_velocity[dim];
+    SparseDirectUMFPACK prec_mass, prec_pressure;
+
+    ConvergenceTable convergence_table;
+
+    DeclException2( ExcInvalidTimeStep, double, double, <<" The time step "<<arg1<<" is out of range."<<std::endl
+                                                                <<" The permitted range is (0,"<<arg2<<"]");
+
+    void Create_Triangulation( const unsigned int n_of_refines );
+
+    inline void interpolate_velocity();
+    inline void diffusion_step( const bool reinit_prec );
+    inline void projection_step( const bool reinit_prec );
+    inline void update_pressure( const bool reinit_prec );
+  private:
+    unsigned int vel_max_its, vel_Krylov_size, vel_off_diagonals, vel_update_prec;
+    double vel_eps, vel_diag_strength;
+    unsigned int proj_max_its, proj_off_diagonals;
+    double proj_eps, proj_diag_strength;
+
+    unsigned int pres_max_its, pres_off_diagonals;
+    double pres_eps, pres_diag_strength;
+
+    inline void init_velocity_matrices();
+    inline void init_pressure_matrices();
+    inline void init_gradient_operator();
+
+    typedef std_cxx1x::tuple< typename DoFHandler<dim>::active_cell_iterator,
+                                                 typename DoFHandler<dim>::active_cell_iterator
+                                               > IteratorTuple;
+    typedef parallel::internal::SynchronousIterators<IteratorTuple> SIterators;
+    struct InitGradPerTaskData{
+      unsigned int d, vel_dpc, pres_dpc;
+      FullMatrix<double> local_grad;
+      std::vector<unsigned int> vel_local_dof_indices, pres_local_dof_indices;
+      InitGradPerTaskData( const unsigned int dd, const unsigned int vdpc, const unsigned int pdpc ):
+                                             d(dd), vel_dpc( vdpc ), pres_dpc( pdpc ),
+                                             local_grad( vdpc, pdpc ), vel_local_dof_indices( vdpc ),
+                                             pres_local_dof_indices( pdpc ){}
+    };
+    struct InitGradScratchData{
+      unsigned int nqp;
+      FEValues<dim> fe_val_vel, fe_val_pres;
+      InitGradScratchData( const FE_Q<dim> &fe_v, const FE_Q<dim> &fe_p, const QGauss<dim> &quad,
+                           const UpdateFlags flags_v, const UpdateFlags flags_p ) :
+                              nqp( quad.size() ), fe_val_vel( fe_v, quad, flags_v ),
+                              fe_val_pres( fe_p, quad, flags_p ){}
+      InitGradScratchData( const InitGradScratchData &data ): nqp( data.nqp ),
+                                        fe_val_vel( data.fe_val_vel.get_fe(), data.fe_val_vel.get_quadrature(),
+                                                    data.fe_val_vel.get_update_flags() ),
+                                        fe_val_pres( data.fe_val_pres.get_fe(), data.fe_val_pres.get_quadrature(),
+                                                     data.fe_val_pres.get_update_flags() ) {}
+    };
+    void assemble_one_cell_of_gradient( const SIterators &SI, InitGradScratchData &scratch,
+                                        InitGradPerTaskData &data );
+    void copy_gradient_local_to_global( const InitGradPerTaskData &data );
+
+    inline void assemble_advection_term();
+    struct AdvectionPerTaskData{
+      FullMatrix<double> local_advection;
+      std::vector<unsigned int> local_dof_indices;
+      AdvectionPerTaskData( const unsigned int dpc ): local_advection( dpc, dpc ), local_dof_indices( dpc ) {}
+    };
+    struct AdvectionScratchData{
+      unsigned int nqp, dpc;
+      std::vector< Point<dim> > u_star_local;
+      std::vector< Tensor<1,dim> > grad_u_star;
+      std::vector<double> u_star_tmp;
+      FEValues<dim> fe_val;
+      AdvectionScratchData( const FE_Q<dim> &fe, const QGauss<dim> &quad, const UpdateFlags flags ):
+                            nqp( quad.size() ), dpc( fe.dofs_per_cell ),
+                            u_star_local( nqp ), grad_u_star( nqp ), u_star_tmp( nqp ),
+                            fe_val( fe, quad, flags ){}
+      AdvectionScratchData( const AdvectionScratchData &data ): nqp( data.nqp ), dpc( data.dpc ),
+                            u_star_local( nqp ), grad_u_star( nqp ), u_star_tmp( nqp ),
+                            fe_val( data.fe_val.get_fe(), data.fe_val.get_quadrature(),
+                                    data.fe_val.get_update_flags() ) {}
+    };
+
+    void assemble_one_cell_of_advection( const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                         AdvectionScratchData &scratch, AdvectionPerTaskData &data );
+    void copy_advection_local_to_global( const AdvectionPerTaskData &data );
+    inline void diffusion_component_solve( const unsigned int d );
+
+    inline void plot_solution( const unsigned int step );
+};
+
+
+template<int dim> Navier_Stokes_Projection<dim>::~Navier_Stokes_Projection(){
+  dof_handler_velocity.clear();
+  dof_handler_pressure.clear();
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::set_dt( const double ddt ){
+  AssertThrow( not ( ( ddt <= 0. ) or ( ddt > .5*T ) ), ExcInvalidTimeStep( ddt, .5*T ) );
+  dt = ddt;
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::Navier_Stokes_Projection</code> }
+// In the constructor, we just read all the data from the <code>Data_Storage</code>
+// object that is passed as an argument, verify that the read data is reasonable
+// and, finally, create the triangulation and load the initial data.
+template<int dim> Navier_Stokes_Projection<dim>::Navier_Stokes_Projection(
+                                        const RunTimeParameters::Data_Storage &data ):
+                    type( data.form ), deg( data.pressure_degree ), dt( data.initial_dt ), t_0( data.initial_time ),
+                    T( data.final_time ), Re( data.Reynolds ), rhs( data.initial_time ),
+                    vel_exact( data.initial_time ), dof_handler_velocity( triangulation ),
+                    dof_handler_pressure( triangulation ), fe_velocity( deg+1 ), fe_pressure( deg ),
+                    quadrature_pressure( deg+1 ), quadrature_velocity( deg+2 ),
+                    vel_max_its( data.vel_max_iterations ), vel_Krylov_size( data.vel_Krylov_size ),
+                    vel_off_diagonals( data.vel_off_diagonals ),
+                    vel_update_prec( data.vel_update_prec ), vel_eps( data.vel_eps ),
+                    vel_diag_strength( data.vel_diag_strength),
+                    proj_max_its( data.proj_max_iterations ), proj_off_diagonals( data.proj_off_diagonals ),
+                    proj_eps( data.proj_eps ), proj_diag_strength( data.proj_diag_strength ),
+                    pres_max_its( data.pres_max_iterations), pres_off_diagonals( data.pres_off_diagonals ),
+                    pres_eps( data.pres_eps ), pres_diag_strength( data.pres_diag_strength )
+{
+  if(deg < 1)
+  std::cout<<" WARNING: The chosen pair of finite element spaces is not stable."<<std::endl
+           <<" The obtained results will be nonsense"<<std::endl;
+
+  AssertThrow( not ( ( dt <= 0. ) or ( dt > .5*T ) ), ExcInvalidTimeStep( dt, .5*T ) );
+
+  Create_Triangulation( data.n_of_global_refines );
+  Initialize();
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::Create_Triangulation</code> }
+// The method that creates the triangulation and refines it the needed number of times.
+// After creating the triangulation, it creates the mesh dependent data, i.e. it distributes
+// degrees of freedom and renumbers them, and initializes the matrices and vectors
+// that we will use.
+template<int dim> void Navier_Stokes_Projection<dim>::Create_Triangulation( const unsigned int n_of_refines ){
+  GridGenerator::hyper_ball( triangulation );
+  static const HyperBallBoundary<dim> boundary;
+  triangulation.set_boundary( 0, boundary );
+
+  triangulation.refine_global( n_of_refines );
+  std::cout<<" Number of active cells: "<<triangulation.n_active_cells()<<std::endl;
+
+  dof_handler_velocity.distribute_dofs( fe_velocity );
+  DoFRenumbering::boost::Cuthill_McKee( dof_handler_velocity );
+  dof_handler_pressure.distribute_dofs( fe_pressure );
+  DoFRenumbering::boost::Cuthill_McKee( dof_handler_pressure );
+
+  init_velocity_matrices();
+  init_pressure_matrices();
+  init_gradient_operator();
+
+
+  pres_n.reinit( dof_handler_pressure.n_dofs() );
+  pres_n_minus_1.reinit( dof_handler_pressure.n_dofs() );
+  phi_n.reinit( dof_handler_pressure.n_dofs() );
+  phi_n_minus_1.reinit( dof_handler_pressure.n_dofs() );
+  pres_tmp.reinit( dof_handler_pressure.n_dofs() );
+  for(unsigned int d=0; d<dim; ++d){
+    u_n[d].reinit( dof_handler_velocity.n_dofs() );
+    u_n_minus_1[d].reinit( dof_handler_velocity.n_dofs() );
+    u_star[d].reinit( dof_handler_velocity.n_dofs() );
+    force[d].reinit( dof_handler_velocity.n_dofs() );
+  }
+  v_tmp.reinit( dof_handler_velocity.n_dofs() );
+
+  std::cout<<" dim( X_h ) = "<<( dof_handler_velocity.n_dofs()*dim )<<std::endl
+           <<" dim( M_h ) = "<<dof_handler_pressure.n_dofs()<<std::endl;
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::Initialize</code> }
+// This method creates the constant matrices and loads the initial data
+template<int dim> void Navier_Stokes_Projection<dim>::Initialize(){
+  vel_Laplace_plus_Mass = 0.;
+  vel_Laplace_plus_Mass.add( 1./Re, vel_Laplace );
+  vel_Laplace_plus_Mass.add( 1.5/dt, vel_Mass );
+
+  EquationData::Pressure<dim> pres( t_0 );
+  VectorTools::interpolate( dof_handler_pressure, pres, pres_n_minus_1 );
+  pres.advance_time( dt );
+  VectorTools::interpolate( dof_handler_pressure, pres, pres_n );
+  phi_n = 0.;
+  phi_n_minus_1 = 0.;
+  for(unsigned int d=0; d<dim; ++d){
+    vel_exact.set_time( t_0 );
+    vel_exact.set_component(d);
+    VectorTools::interpolate( dof_handler_velocity, vel_exact, u_n_minus_1[d] );
+    vel_exact.advance_time( dt );
+    VectorTools::interpolate( dof_handler_velocity, vel_exact, u_n[d] );
+  }
+}
+
+
+// @sect4{ The <code>Navier_Stokes_Projection::init_*_matrices</code> methods }
+// In this set of methods we initialize the sparsity patterns, the constraints (if any)
+// and assemble the matrices that do not depend on the timestep $\Delta t$.
+template<int dim> void Navier_Stokes_Projection<dim>::init_velocity_matrices(){
+  spar_pattern_velocity.reinit( dof_handler_velocity.n_dofs(), dof_handler_velocity.n_dofs(),
+                                dof_handler_velocity.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_velocity, spar_pattern_velocity );
+  spar_pattern_velocity.compress();
+
+  vel_Laplace_plus_Mass.reinit( spar_pattern_velocity );
+  for( unsigned int d=0; d<dim; ++d )
+    vel_it_matrix[d].reinit( spar_pattern_velocity );
+  vel_Mass.reinit( spar_pattern_velocity );
+  vel_Laplace.reinit( spar_pattern_velocity );
+  vel_Advection.reinit( spar_pattern_velocity );
+
+  MatrixCreator::create_mass_matrix( dof_handler_velocity, quadrature_velocity, vel_Mass );
+  MatrixCreator::create_laplace_matrix( dof_handler_velocity, quadrature_velocity, vel_Laplace );
+}
+
+// For the initialization of the matrices that act on the pressure space it is worth noticing
+// one small detail. Since the projection step involves the solution
+// of a Poisson equation with homogeneous Neumann boundary conditions, we need somehow to
+// regularize this problem, that is to pick a solution.
+// The way we do it is by setting the value of the solution at the first node (wherever it
+// is) to zero. This regularizes the problem and does not increase the sparsity pattern we
+// use.
+template<int dim> void Navier_Stokes_Projection<dim>::init_pressure_matrices(){
+  spar_pattern_pressure.reinit( dof_handler_pressure.n_dofs(), dof_handler_pressure.n_dofs(),
+                                dof_handler_pressure.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_pressure, spar_pattern_pressure );
+
+  pres_regularization.clear();
+  pres_regularization.add_line(0);
+  pres_regularization.close();
+  pres_regularization.condense( spar_pattern_pressure );
+
+  spar_pattern_pressure.compress();
+
+  pres_Laplace.reinit( spar_pattern_pressure );
+  pres_Mass.reinit( spar_pattern_pressure );
+
+  MatrixCreator::create_laplace_matrix( dof_handler_pressure, quadrature_pressure, pres_Laplace );
+  MatrixCreator::create_mass_matrix(  dof_handler_pressure, quadrature_pressure, pres_Mass );
+
+  pres_regularization.condense( pres_Laplace );
+}
+
+
+// For the gradient operator, we start by initializing the sparsity pattern and compressing it.
+// It is important to notice here that the gradient operator acts from the pressure space
+// into the velocity space, so we have to deal with two different finite element spaces. To keep
+// the loops synchronized, we use the <code>typedef</code>'s that we have defined before,namely
+// <code>PairedIterators</code> and <code>SIterators</code>.
+template<int dim> void Navier_Stokes_Projection<dim>::init_gradient_operator(){
+  spar_pattern_pres_vel.reinit( dof_handler_velocity.n_dofs(), dof_handler_pressure.n_dofs(),
+                                dof_handler_velocity.max_couplings_between_dofs() );
+  DoFTools::make_sparsity_pattern( dof_handler_velocity, dof_handler_pressure, spar_pattern_pres_vel );
+  spar_pattern_pres_vel.compress();
+
+  InitGradPerTaskData per_task_data( 0, fe_velocity.dofs_per_cell, fe_pressure.dofs_per_cell );
+  InitGradScratchData scratch_data( fe_velocity, fe_pressure, quadrature_velocity,
+                                    update_values | update_JxW_values, update_gradients );
+
+  for( unsigned int d=0; d<dim; ++d ){
+    pres_Diff[d].reinit( spar_pattern_pres_vel );
+    per_task_data.d = d;
+    WorkStream::run( SIterators( IteratorTuple( dof_handler_velocity.begin_active(),
+                                                dof_handler_pressure.begin_active()
+                                              )
+                               ),
+                     SIterators( IteratorTuple( dof_handler_velocity.end(),
+                                                dof_handler_pressure.end()
+                                              )
+                               ),
+                     *this,
+                     &Navier_Stokes_Projection<dim>::assemble_one_cell_of_gradient,
+                     &Navier_Stokes_Projection<dim>::copy_gradient_local_to_global,
+                     scratch_data,
+                     per_task_data
+                   );
+  }
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::assemble_one_cell_of_gradient( const SIterators &SI,
+                                                              InitGradScratchData &scratch,
+                                                              InitGradPerTaskData &data ){
+  scratch.fe_val_vel.reinit( std_cxx1x::get<0>( SI.iterators ) );
+  scratch.fe_val_pres.reinit( std_cxx1x::get<1>( SI.iterators ) );
+
+  std_cxx1x::get<0>( SI.iterators )->get_dof_indices( data.vel_local_dof_indices );
+  std_cxx1x::get<1>( SI.iterators )->get_dof_indices( data.pres_local_dof_indices );
+
+  data.local_grad = 0.;
+  for( unsigned int q=0; q<scratch.nqp; ++q ){
+    for( unsigned int i=0; i<data.vel_dpc; ++i )
+      for( unsigned int j=0; j<data.pres_dpc; ++j )
+        data.local_grad( i, j ) += scratch.fe_val_vel.JxW(q)*scratch.fe_val_vel.shape_value( i, q )
+                                   *scratch.fe_val_pres.shape_grad( j, q )[data.d];
+  }
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::copy_gradient_local_to_global(
+                                                                  const InitGradPerTaskData &data ){
+  for( unsigned int i=0; i<data.vel_dpc; ++i )
+    for( unsigned int j=0; j<data.pres_dpc; ++j)
+      pres_Diff[data.d].add( data.vel_local_dof_indices[i], data.pres_local_dof_indices[j],
+                             data.local_grad( i, j)  );
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::run</code> }
+// This is the time marching function, which starting at <code>t_0</code> advances in time
+// using the projection method with time step <code>dt</code> until <code>T</code>. <br>
+// The boolean parameter, <code>verbose</code>, that it takes is to enable
+// information about what the method is doing at the given moment, i.e. diffusion, projection
+// substep; updating preconditioners etc. This is useful mostly for debugging purposes
+// and so it is by default set to false
+template<int dim> void Navier_Stokes_Projection<dim>::run( const bool verbose, const unsigned int n_of_plots ){
+  unsigned int n_steps = ( T - t_0 )/dt;
+  rhs.set_time( 2.*dt );
+  vel_exact.set_time( 2.*dt );
+
+  for( unsigned int n = 2; n<=n_steps; ++n ){
+    if( verbose )
+      std::cout<<" Step = "<<n<<" Time = "<<(n*dt)<<std::endl;
+    if( verbose )
+      std::cout<<"  Interpolating the velocity "<<std::endl;
+    interpolate_velocity();
+    if( verbose )
+      std::cout<<"  Diffusion Step"<<std::endl;
+    if( ( n%vel_update_prec == 0 ) and ( verbose ) )
+      std::cout<<"   With reinitialization of the preconditioner"<<std::endl;
+    diffusion_step( (n%vel_update_prec == 0 ) or ( n == 2) );
+    if( verbose )
+      std::cout<<"  Projection Step"<<std::endl;
+    projection_step( ( n == 2 ) );
+    if( verbose )
+      std::cout<<"  Updating the Pressure"<<std::endl;
+    update_pressure( ( n == 2 ) );
+    if( n%n_of_plots == 0 ){
+      if( verbose )
+        std::cout<<" Plotting Solution"<<std::endl;
+      plot_solution(n);
+    }
+    rhs.advance_time(dt);
+    vel_exact.advance_time(dt);
+  }
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::interpolate_velocity(){
+  for( unsigned int d=0; d<dim; ++d ){
+    u_star[d] = 0.;
+    u_star[d].add( 2., u_n[d], -1., u_n_minus_1[d] );
+  }
+}
+
+
+// @sect4{<code>Navier_Stokes_Projection::diffusion_step</code>}
+// The implementation of a diffusion step.
+template<int dim> void Navier_Stokes_Projection<dim>::diffusion_step( const bool reinit_prec ){
+  pres_tmp = pres_n;
+  pres_tmp.add(4./3., phi_n, -1./3., phi_n_minus_1);
+  pres_tmp *= -1.;
+
+  assemble_advection_term();
+
+  for( unsigned int d=0; d<dim; ++d ){
+    rhs.set_component(d);
+    VectorTools::create_right_hand_side( dof_handler_velocity, quadrature_velocity, rhs, force[d] );
+
+    v_tmp = 0.;
+    v_tmp.add( 2./dt,u_n[d],-.5/dt,u_n_minus_1[d] );
+    vel_Mass.vmult_add( force[d], v_tmp );
+
+    pres_Diff[d].vmult_add( force[d], pres_tmp );
+    u_n_minus_1[d] = u_n[d];
+
+    vel_it_matrix[d].copy_from( vel_Laplace_plus_Mass );
+    vel_it_matrix[d].add( 1., vel_Advection );
+
+    vel_exact.set_component(d);
+    VectorTools::interpolate_boundary_values( dof_handler_velocity, 0, vel_exact, boundary_values );
+    MatrixTools::apply_boundary_values( boundary_values, vel_it_matrix[d], u_n[d], force[d] );
+  }
+
+  Threads::TaskGroup<void> tasks;
+  for(unsigned int d=0; d<dim; ++d ){
+    if( reinit_prec )
+      prec_velocity[d].initialize( vel_it_matrix[d],
+                                  SparseILU<double>::AdditionalData( vel_diag_strength, vel_off_diagonals ) );
+   tasks += Threads::new_task( &Navier_Stokes_Projection<dim>::diffusion_component_solve, *this, d );
+  }
+  tasks.join_all();
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::diffusion_component_solve( const unsigned int d){
+  SolverControl solver_control( vel_max_its, vel_eps*force[d].l2_norm() );
+  SolverGMRES<> gmres( solver_control, SolverGMRES<>::AdditionalData() );
+  gmres.solve( vel_it_matrix[d], u_n[d], force[d], prec_velocity[d] );
+}
+
+
+// @sect4{ The <code>Navier_Stokes_Projection::assemble_advection_term</code>  method and related}
+template<int dim> void Navier_Stokes_Projection<dim>::assemble_advection_term(){
+  vel_Advection = 0.;
+  AdvectionPerTaskData data( fe_velocity.dofs_per_cell );
+  AdvectionScratchData scratch( fe_velocity, quadrature_velocity,
+                                update_values | update_JxW_values | update_gradients );
+  WorkStream::run( dof_handler_velocity.begin_active(), dof_handler_velocity.end(), *this,
+                   &Navier_Stokes_Projection<dim>::assemble_one_cell_of_advection,
+                   &Navier_Stokes_Projection<dim>::copy_advection_local_to_global, scratch, data);
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::assemble_one_cell_of_advection(
+                                                  const typename DoFHandler<dim>::active_cell_iterator &cell,
+                                                  AdvectionScratchData &scratch, AdvectionPerTaskData &data ){
+  scratch.fe_val.reinit(cell);
+  cell->get_dof_indices( data.local_dof_indices );
+  for( unsigned int d=0; d<dim; ++d ){
+    scratch.fe_val.get_function_values( u_star[d], scratch.u_star_tmp );
+    for( unsigned int q=0; q<scratch.nqp; ++q )
+      scratch.u_star_local[q](d) = scratch.u_star_tmp[q];
+  }
+
+  for( unsigned int d=0; d<dim; ++d ){
+    scratch.fe_val.get_function_gradients( u_star[d], scratch.grad_u_star );
+    for( unsigned int q=0; q<scratch.nqp; ++q ){
+      if( d==0 )
+        scratch.u_star_tmp[q] = 0.;
+      scratch.u_star_tmp[q] += scratch.grad_u_star[q][d];
+    }
+  }
+
+  data.local_advection = 0.;
+  for( unsigned int q=0; q<scratch.nqp; ++q )
+    for( unsigned int i=0; i<scratch.dpc; ++i )
+      for( unsigned int j=0; j<scratch.dpc; ++j )
+        data.local_advection(i,j) += (
+                scratch.u_star_local[q]*scratch.fe_val.shape_grad( j, q )*scratch.fe_val.shape_value( i, q )
+                + 0.5*scratch.u_star_tmp[q]*scratch.fe_val.shape_value( i, q )*scratch.fe_val.shape_value( j, q )
+                                    )*scratch.fe_val.JxW(q) ;
+}
+
+template<int dim> void Navier_Stokes_Projection<dim>::copy_advection_local_to_global(
+                                                        const AdvectionPerTaskData &data ){
+  for( unsigned int i=0; i<fe_velocity.dofs_per_cell; ++i )
+    for( unsigned int j=0; j<fe_velocity.dofs_per_cell; ++j )
+      vel_Advection.add( data.local_dof_indices[i] , data.local_dof_indices[j], data.local_advection(i,j) );
+}
+
+
+
+
+// @sect4{<code>Navier_Stokes_Projection::projection_step</code>}
+// This implements the projection step.
+template<int dim> void Navier_Stokes_Projection<dim>::projection_step( const bool reinit_prec ){
+  if( reinit_prec )
+    prec_pressure.initialize( pres_Laplace );
+
+  pres_tmp = 0.;
+  for( unsigned d=0; d<dim; ++d )
+    pres_Diff[d].Tvmult_add( pres_tmp, u_n[d] );
+
+  phi_n_minus_1 = phi_n;
+  phi_n = pres_tmp;
+  phi_n *= 1.5/dt;
+
+  pres_regularization.condense( phi_n );
+  prec_pressure.solve( phi_n );
+  pres_regularization.distribute( phi_n );
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::update_pressure</code> }
+// This is the pressure update step of the projection method. It implements the
+// standard formulation of the method, that is
+// $$
+//      p^{n+1} = p^n + \phi^{n+1},
+// $$
+// or the rotational form, which is
+// $$
+//      p^{n+1} = p^n + \phi^{n+1} - \frac{1}{Re} \nabla\cdot u^{n+1}.
+// $$
+template<int dim> void Navier_Stokes_Projection<dim>::update_pressure( const bool reinit_prec ){
+  pres_n_minus_1 = pres_n;
+  switch( type ){
+    case RunTimeParameters::METHOD_STANDARD:
+      pres_n += phi_n;
+      break;
+    case RunTimeParameters::METHOD_ROTATIONAL:
+      if( reinit_prec )
+        prec_mass.initialize( pres_Mass );
+      pres_n = pres_tmp;
+      prec_mass.solve( pres_n );
+      pres_n.sadd(1./Re, 1., pres_n_minus_1, 1., phi_n );
+      break;
+    default:
+      Assert( false, ExcNotImplemented() );
+  };
+}
+
+
+// @sect4{ <code>Navier_Stokes_Projection::plot_solution</code> }
+// At this stage, we only output the vorticity of the flow. This only works in 2d and
+// WILL be changed.
+///
+template<int dim> void Navier_Stokes_Projection<dim>::plot_solution( const unsigned int step ){
+  const FESystem<dim> joint_fe( fe_velocity, dim, fe_pressure, 1 );
+  DoFHandler<dim> joint_dof_handler( triangulation );
+  joint_dof_handler.distribute_dofs( joint_fe );
+  Assert( joint_dof_handler.n_dofs() == dim*dof_handler_velocity.n_dofs() + dof_handler_pressure.n_dofs(),
+          ExcInternalError() );
+  static Vector<double> joint_solution( joint_dof_handler.n_dofs() );
+  std::vector<unsigned int> loc_joint_dof_indices( joint_fe.dofs_per_cell ),
+                            loc_vel_dof_indices( fe_velocity.dofs_per_cell ),
+                            loc_pres_dof_indices( fe_pressure.dofs_per_cell );
+  typename DoFHandler<dim>::active_cell_iterator
+                              joint_cell = joint_dof_handler.begin_active(),
+                              joint_endc = joint_dof_handler.end(),
+                              vel_cell   = dof_handler_velocity.begin_active(),
+                              pres_cell  = dof_handler_pressure.begin_active();
+  for( ; joint_cell not_eq joint_endc; ++joint_cell, ++vel_cell, ++pres_cell ){
+    joint_cell->get_dof_indices( loc_joint_dof_indices );
+    vel_cell->get_dof_indices( loc_vel_dof_indices ),
+    pres_cell->get_dof_indices( loc_pres_dof_indices );
+    for( unsigned int i=0; i<joint_fe.dofs_per_cell; ++i )
+      switch( joint_fe.system_to_base_index(i).first.first ){
+        case 0:
+          // Velocity
+          Assert( joint_fe.system_to_base_index(i).first.second < dim, ExcInternalError() );
+          joint_solution( loc_joint_dof_indices[i] ) =
+                              u_n[ joint_fe.system_to_base_index(i).first.second ]
+                                      ( loc_vel_dof_indices[ joint_fe.system_to_base_index(i).second ] );
+
+          break;
+        case 1:
+          // Pressure
+          Assert( joint_fe.system_to_base_index(i).first.second == 0, ExcInternalError() );
+          joint_solution( loc_joint_dof_indices[i] ) =
+                                          pres_n( loc_pres_dof_indices[ joint_fe.system_to_base_index(i).second ] );
+          break;
+        default:
+          Assert( false, ExcInternalError() );
+      }
+  }
+
+  std::vector<std::string> joint_solution_names( dim, "v" );
+  joint_solution_names.push_back( "p" );
+
+  DataOut<dim> data_out;
+  data_out.attach_dof_handler (joint_dof_handler);
+
+  std::vector< DataComponentInterpretation::DataComponentInterpretation >
+            component_interpretation( dim+1, DataComponentInterpretation::component_is_part_of_vector );
+  component_interpretation[dim] = DataComponentInterpretation::component_is_scalar;
+
+  data_out.add_data_vector( joint_solution, joint_solution_names, DataOut<dim>::type_dof_data,
+                            component_interpretation );
+
+  data_out.build_patches( deg + 1 );
+
+  std::ostringstream filename;
+  filename<<"solution-"<<step<<".vtk";
+
+  std::ofstream output( filename.str().c_str() );
+  data_out.write_vtk( output );
+}
+
+
+// @sect4{<code>Navier_Stokes_Projection::Post_Process</code>}
+// Having reached the final time <code>T</code>, we want to measure the error that we have made.
+// This method is responsible for that. Saves the results in a <code>ConvergenceTable</code>
+// object which later we can print or compute things with it. <br>
+// The way we compute the errors is very similar to previous tutorials. However, we need the
+// pressure to have mean value zero, so we compute its mean value and subtract it from the computed
+// pressure.
+template<int dim> void Navier_Stokes_Projection<dim>::Post_Process(){
+  double tmp, vel_err_L2=0., vel_err_H1=0., pres_err_L2;
+
+  Vector<double> differences( triangulation.n_active_cells() );
+
+  vel_exact.set_time(T);
+  for( unsigned int d=0; d<dim; ++d ){
+    vel_exact.set_component(d);
+
+    differences = 0.;
+    VectorTools::integrate_difference( dof_handler_velocity, u_n[d], vel_exact, differences,
+                                       quadrature_velocity, VectorTools::L2_norm );
+    tmp = differences.l2_norm();
+    vel_err_L2 += tmp*tmp;
+
+    differences = 0.;
+    VectorTools::integrate_difference( dof_handler_velocity, u_n[d], vel_exact, differences,
+                                       quadrature_velocity, VectorTools::H1_seminorm );
+    tmp = differences.l2_norm();
+    vel_err_H1 += tmp*tmp;
+  }
+  vel_err_L2 = std::sqrt( vel_err_L2 );
+  vel_err_H1 = std::sqrt( vel_err_H1 );
+
+  double pres_mean_value = VectorTools::compute_mean_value( dof_handler_pressure, quadrature_pressure, pres_n, 0 );
+  pres_n.add( -pres_mean_value );
+  EquationData::Pressure<dim> pres_exact(T);
+  differences = 0.;
+  VectorTools::integrate_difference( dof_handler_pressure, pres_n, pres_exact,
+                                     differences, quadrature_pressure, VectorTools::L2_norm );
+  pres_err_L2 = differences.l2_norm();
+
+  convergence_table.add_value( "dt"     , dt );
+  convergence_table.add_value( "u_L2"   , vel_err_L2 );
+  convergence_table.add_value( "u_H1"   , vel_err_H1 );
+  convergence_table.add_value( "pres_L2", pres_err_L2 );
+
+  convergence_table.set_precision( "dt"     , 5 );
+  convergence_table.set_precision( "u_L2"   , 5 );
+  convergence_table.set_precision( "u_H1"   , 5 );
+  convergence_table.set_precision( "pres_L2", 5 );
+
+  convergence_table.set_scientific( "u_L2"   , true );
+  convergence_table.set_scientific( "u_H1"   , true );
+  convergence_table.set_scientific( "pres_L2", true );
+
+  convergence_table.write_text(std::cout);
+}
+
+
+
+// @sect3{ The main function }
+// The main function looks very much like in all the other tutorial programs.
+int main(){
+  try{
+    RunTimeParameters::Data_Storage data;
+    data.read_data( "parameter-file.prm" );
+    deallog.depth_console( data.verbose?2:0 );
+    Navier_Stokes_Projection<2> test( data );
+    for( double dt = data.initial_dt; dt >= data.final_dt; dt /= data.dt_decrement ){
+      std::cout<<" dt = "<<dt<<std::endl;
+      test.set_dt( dt );
+      test.Initialize();
+      test.run( data.verbose, data.output );
+      test.Post_Process();
+      std::cout<<"====================================="<<std::endl<<std::endl;
+    }
+  }
+  catch (std::exception &exc){
+    std::cerr << std::endl << std::endl
+            << "----------------------------------------------------"
+            << std::endl;
+    std::cerr << "Exception on processing: " << std::endl
+            << exc.what() << std::endl
+            << "Aborting!" << std::endl
+            << "----------------------------------------------------"
+            << std::endl;
+    return 1;
+  }
+  catch (...){
+    std::cerr << std::endl << std::endl
+              << "----------------------------------------------------"
+              << std::endl;
+    std::cerr << "Unknown exception!" << std::endl
+              << "Aborting!" << std::endl
+              << "----------------------------------------------------"
+              << std::endl;
+    return 1;
+  }
+  std::cout<<"----------------------------------------------------"
+           <<std::endl
+           <<"Apparently everything went fine!"
+           <<std::endl
+           <<"Don't forget to brush your teeth :-)"
+           <<std::endl<<std::endl;
+  return 0;
+}
index ea368148a0916d8fe73ade637ec3ee6e6aba24e3..026426c6fca1171200a4eb233c0b50e14c68a02a 100644 (file)
 #include <fe/fe_q.h>
 #include <grid/grid_out.h>
 
-                                // The following two files provide
-                                // classes and information for
-                                // multi-threaded programs. In the
-                                // first one, the classes and
-                                // functions are declared which we
-                                // need to start new threads and to
-                                // wait for threads to return
-                                // (i.e. the <code>Thread</code> class
-                                // and the <code>spawn</code> functions). The
+                                // The following two files provide classes
+                                // and information for multi-threaded
+                                // programs. In the first one, the classes
+                                // and functions are declared which we need
+                                // to start new threads and to wait for
+                                // threads to return (i.e. the
+                                // <code>Thread</code> class and the
+                                // <code>new_thread</code> functions). The
                                 // second file has a class
                                 // <code>MultithreadInfo</code> (and a global
                                 // object <code>multithread_info</code> of
-                                // that type) which can be used to
-                                // query the number of processors in
-                                // your system, which is often useful
-                                // when deciding how many threads to
-                                // start in parallel.
+                                // that type) which can be used to query the
+                                // number of processors in your system, which
+                                // is often useful when deciding how many
+                                // threads to start in parallel.
 #include <base/thread_management.h>
 #include <base/multithread_info.h>
 
@@ -122,6 +120,18 @@ class AdvectionProblem
                                     // which denote the first cell on
                                     // which it shall operate, and
                                     // the one past the last.
+                                    //
+                                    // The strategy for parallelization we
+                                    // choose here is one of the
+                                    // possibilities mentioned in detail in
+                                    // the @ref threads module in the
+                                    // documentation. While it is a
+                                    // straightforward way to distribute the
+                                    // work for assembling the system onto
+                                    // multiple processor cores. As mentioned
+                                    // in the module, there are other, and
+                                    // possibly better suited, ways to
+                                    // achieve the same goal.
     void assemble_system ();
     void assemble_system_interval (const typename DoFHandler<dim>::active_cell_iterator &begin,
                                   const typename DoFHandler<dim>::active_cell_iterator &end);
@@ -733,26 +743,39 @@ void AdvectionProblem<dim>::assemble_system ()
                                   // another value. This variable is
                                   // also queried by functions inside
                                   // the library to determine how
-                                  // many threads they shall spawn.
+                                  // many threads they shall create.
   const unsigned int n_threads = multithread_info.n_default_threads;
+                                  // It is worth noting, however, that this
+                                  // setup determines the load distribution
+                                  // onto processor in a static way: it does
+                                  // not take into account that some other
+                                  // part of our program may also be running
+                                  // something in parallel at the same time
+                                  // as we get here (this is not the case in
+                                  // the current program, but may easily be
+                                  // the case in more complex
+                                  // applications). A discussion of how to
+                                  // deal with this case can be found in the
+                                  // @ref threads module.
+                                  //
                                   // Next, we need an object which is
                                   // capable of keeping track of the
                                   // threads we created, and allows
                                   // us to wait until they all have
                                   // finished (to <code>join</code> them in
                                   // the language of threads). The
-                                  // <code>Threads::ThreadGroup</code> class
+                                  // Threads::ThreadGroup class
                                   // does this, which is basically
                                   // just a container for objects of
-                                  // type <code>Threads::Thread</code> that
+                                  // type Threads::Thread that
                                   // represent a single thread;
-                                  // <code>Threads::Thread</code> is what the
-                                  // <code>spawn</code> function below will
+                                  // Threads::Thread is what the
+                                  // Threads::new_thread function below will
                                   // return when we start a new
                                   // thread.
                                   //
-                                  // Note that both <code>ThreadGroup</code>
-                                  // and <code>Thread</code> have a template
+                                  // Note that both Threads::ThreadGroup
+                                  // and Threads::Thread have a template
                                   // argument that represents the
                                   // return type of the function
                                   // being called on a separate
@@ -769,7 +792,7 @@ void AdvectionProblem<dim>::assemble_system ()
                                   //
                                   // If you did not configure for
                                   // multi-threading, then the
-                                  // <code>spawn</code> function that is
+                                  // <code>new_thread</code> function that is
                                   // supposed to start a new thread
                                   // in parallel only executes the
                                   // function which should be run in
@@ -781,7 +804,7 @@ void AdvectionProblem<dim>::assemble_system ()
                                   // <code>join</code> that is supposed to
                                   // wait for all spawned threads to
                                   // return, returns immediately, as
-                                  // there can't be threads running.
+                                  // there can't be any threads running.
   Threads::ThreadGroup<> threads;
 
                                   // Now we have to split the range
@@ -848,24 +871,25 @@ void AdvectionProblem<dim>::assemble_system ()
                                   // using the following sequence of
                                   // function calls:
   for (unsigned int thread=0; thread<n_threads; ++thread)
-    threads += Threads::spawn (*this, &AdvectionProblem<dim>::assemble_system_interval)
-               (thread_ranges[thread].first,
-                thread_ranges[thread].second);
+    threads += Threads::new_thread (&AdvectionProblem<dim>::assemble_system_interval,
+                                   *this,
+                                   thread_ranges[thread].first,
+                                   thread_ranges[thread].second);
                                   // The reasons and internal
                                   // workings of these functions can
                                   // be found in the report on the
                                   // subject of multi-threading,
                                   // which is available online as
                                   // well. Suffice it to say that we
-                                  // spawn a new thread that calls
+                                  // create a new thread that calls
                                   // the <code>assemble_system_interval</code>
                                   // function on the present object
                                   // (the <code>this</code> pointer), with the
                                   // arguments following in the
                                   // second set of parentheses passed
-                                  // as parameters. The <code>spawn</code>
-                                  // function return an object of
-                                  // type <code>Threads::Thread</code>, which
+                                  // as parameters. The Threads::new_thread
+                                  // function returns an object of
+                                  // type Threads::Thread, which
                                   // we put into the <code>threads</code>
                                   // container. If a thread exits,
                                   // the return value of the function
@@ -1471,16 +1495,15 @@ GradientEstimation::estimate (const DoFHandler<dim> &dof_handler,
     = Threads::split_interval (0, dof_handler.get_tria().n_active_cells(),
                               n_threads);
 
-                                  // In the same way as before, we
-                                  // use a <code>Threads::ThreadGroup</code>
-                                  // object to collect the descriptor
-                                  // objects of different
-                                  // threads. Note that as the
+                                  // In the same way as before, we use a
+                                  // <code>Threads::ThreadGroup</code> object
+                                  // to collect the descriptor objects of
+                                  // different threads. Note that as the
                                   // function called is not a member
-                                  // function, but rather a static
-                                  // function, we need not (and can
-                                  // not) pass a <code>this</code> pointer to
-                                  // the <code>spawn</code> function in this
+                                  // function, but rather a static function,
+                                  // we need not (and can not) pass a
+                                  // <code>this</code> pointer to the
+                                  // <code>new_thread</code> function in this
                                   // case.
                                   //
                                   // Taking pointers to templated
@@ -1508,9 +1531,10 @@ GradientEstimation::estimate (const DoFHandler<dim> &dof_handler,
                                 Vector<float> &)
     = &GradientEstimation::template estimate_interval<dim>;
   for (unsigned int i=0; i<n_threads; ++i)
-    threads += Threads::spawn (estimate_interval_ptr)(dof_handler, solution,
-                                                      index_intervals[i],
-                                                      error_per_cell);
+    threads += Threads::new_thread (estimate_interval_ptr,
+                                   dof_handler, solution,
+                                   index_intervals[i],
+                                   error_per_cell);
                                   // Ok, now the threads are at work,
                                   // and we only have to wait for
                                   // them to finish their work:
index 852d875bb16d1379411439a0134f3948189a8feb..f49878b2f9d1afe98c9b2b9130f406d69cbf487c 100644 (file)
@@ -30,12 +30,6 @@ endif
 
 
 # production rules
-ifneq ($(with-multithreading),no)
-  MT = MT
-else
-  MT = ==
-endif
-
 $(LIBDIR)/lac/%.g.$(OBJEXT) :
        @echo "=====lac==============debug======$(MT)== $(<F)"
        @$(CXX) $(CXXFLAGS.g) -c $< -o $@
index e225a76fd4628814b5c8a1112c96fbaae97e478a..221704a33b8662ee3e2b5cfcc61a22996bfa02e2 100644 (file)
@@ -1657,3 +1657,4 @@ ConstraintMatrix::is_constrained (const unsigned int index) const
 DEAL_II_NAMESPACE_CLOSE
 
 #endif
+
index 424b8f895dd6e5b57d01d3777a5f8a1c66156fa3..a269541aa1bdfe101ce864e4243e1c0804d0baf0 100644 (file)
@@ -30,6 +30,21 @@ template<typename number> class FullMatrix;
  *@{
  */
 
+namespace internal
+{
+  namespace SparseMatrix
+  {
+                                    /**
+                                     * Like
+                                     * internal::Vector::minimum_parallel_grain_size,
+                                     * but now denoting the number of rows of
+                                     * a matrix that should be worked on as a
+                                     * minimum.
+                                     */
+    extern unsigned int minimum_parallel_grain_size;
+  }
+}
+
 namespace SparseMatrixIterators
 {
                                   // forward declaration
@@ -2019,75 +2034,6 @@ class SparseMatrix : public virtual Subscriptor
                                      */
     unsigned int max_len;
 
-                                    /**
-                                     * Version of vmult() which only
-                                     * performs its actions on the
-                                     * region defined by
-                                     * <tt>[begin_row,end_row)</tt>. This
-                                     * function is called by vmult()
-                                     * in the case of enabled
-                                     * multithreading.
-                                     */
-    template <class OutVector, class InVector>
-    void threaded_vmult (OutVector& dst,
-                        const InVector& src,
-                        const unsigned int        begin_row,
-                        const unsigned int        end_row) const;
-
-                                    /**
-                                     * Version of
-                                     * matrix_norm_square() which
-                                     * only performs its actions on
-                                     * the region defined by
-                                     * <tt>[begin_row,end_row)</tt>. This
-                                     * function is called by
-                                     * matrix_norm_square() in the
-                                     * case of enabled
-                                     * multithreading.
-                                     */
-    template <typename somenumber>
-    void threaded_matrix_norm_square (const Vector<somenumber> &v,
-                                     const unsigned int        begin_row,
-                                     const unsigned int        end_row,
-                                     somenumber               *partial_sum) const;
-
-                                    /**
-                                     * Version of
-                                     * matrix_scalar_product() which
-                                     * only performs its actions on
-                                     * the region defined by
-                                     * <tt>[begin_row,end_row)</tt>. This
-                                     * function is called by
-                                     * matrix_scalar_product() in the
-                                     * case of enabled
-                                     * multithreading.
-                                     */
-    template <typename somenumber>
-    void threaded_matrix_scalar_product (const Vector<somenumber> &u,
-                                        const Vector<somenumber> &v,
-                                        const unsigned int        begin_row,
-                                        const unsigned int        end_row,
-                                        somenumber               *partial_sum) const;
-
-                                    /**
-                                     * Version of residual() which
-                                     * only performs its actions on
-                                     * the region defined by
-                                     * <tt>[begin_row,end_row)</tt>
-                                     * (these numbers are the
-                                     * components of
-                                     * <tt>interval</tt>). This
-                                     * function is called by
-                                     * residual() in the case of
-                                     * enabled multithreading.
-                                     */
-    template <typename somenumber>
-    void threaded_residual (Vector<somenumber>       &dst,
-                           const Vector<somenumber> &u,
-                           const Vector<somenumber> &b,
-                           const std::pair<unsigned int,unsigned int> interval,
-                           somenumber               *partial_norm) const;
-
                                     // make all other sparse matrices
                                     // friends
     template <typename somenumber> friend class SparseMatrix;
index 84ffb321b11eb9ac659dbb30db2e6fb1f2682b4f..df34ec954645afa55e0fb022983f240f694b5adb 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2009 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 #define __deal2__sparse_matrix_templates_h
 
 
-//TODO[WB]: the threaded functions can now be converted to return a value, rather than use an additional argument
-
+#include <base/config.h>
 #include <base/template_constraints.h>
+#include <base/parallel.h>
+#include <base/thread_management.h>
+#include <base/multithread_info.h>
 #include <lac/sparse_matrix.h>
 #include <lac/vector.h>
 #include <lac/full_matrix.h>
 #include <algorithm>
 #include <functional>
 #include <cmath>
-
 #include <vector>
 #include <numeric>
+#include <base/std_cxx1x/bind.h>
 
-#include <base/thread_management.h>
-#include <base/multithread_info.h>
 
 
 DEAL_II_NAMESPACE_OPEN
@@ -331,6 +331,59 @@ SparseMatrix<number>::add (const number factor,
 }
 
 
+namespace internal
+{
+  namespace SparseMatrix
+  {
+                                    /**
+                                     * Perform a vmult using the SparseMatrix
+                                     * data structures, but only using a
+                                     * subinterval for the row indices.
+                                     *
+                                     * In the sequential case, this function
+                                     * is called on all rows, in the parallel
+                                     * case it may be called on a subrange,
+                                     * at the discretion of the task
+                                     * scheduler.
+                                     */
+    template <typename number,
+             typename InVector,
+             typename OutVector>
+    void vmult_on_subrange (const unsigned int  begin_row,
+                           const unsigned int  end_row,
+                           const number       *values,
+                           const std::size_t  *rowstart,
+                           const unsigned int *colnums,
+                           const InVector     &src,
+                           OutVector          &dst,
+                           const bool          add)
+    {
+      const number       *val_ptr    = &values[rowstart[begin_row]];
+      const unsigned int *colnum_ptr = &colnums[rowstart[begin_row]];
+      typename OutVector::iterator dst_ptr = dst.begin() + begin_row;
+
+      if (add == false)
+       for (unsigned int row=begin_row; row<end_row; ++row)
+         {
+           typename OutVector::value_type s = 0.;
+           const number *const val_end_of_row = &values[rowstart[row+1]];
+           while (val_ptr != val_end_of_row)
+             s += *val_ptr++ * src(*colnum_ptr++);
+           *dst_ptr++ = s;
+         }
+      else
+       for (unsigned int row=begin_row; row<end_row; ++row)
+         {
+           typename OutVector::value_type s = 0.;
+           const number *const val_end_of_row = &values[rowstart[row+1]];
+           while (val_ptr != val_end_of_row)
+             s += *val_ptr++ * src(*colnum_ptr++);
+           *dst_ptr++ += s;
+         }
+    }
+  }
+}
+
 
 template <typename number>
 template <typename number2>
@@ -545,91 +598,18 @@ SparseMatrix<number>::vmult (OutVector& dst,
   Assert(n() == src.size(), ExcDimensionMismatch(n(),src.size()));
 
   Assert (!PointerComparison::equal(&src, &dst), ExcSourceEqualsDestination());
-  
-  const unsigned int n_rows = m();
-
-                                  // in MT mode: start new threads
-                                  // only if the matrix is
-                                  // sufficiently large.  the limit
-                                  // is mostly artificial
-  if (DEAL_II_USE_MT &&
-      (multithread_info.n_default_threads > 1) &&
-      (n_rows/multithread_info.n_default_threads > 2000))
-    {
-      const unsigned int n_threads = multithread_info.n_default_threads;
-
-                                      // then spawn threads. since
-                                      // some compilers have trouble
-                                      // finding out which
-                                      // 'encapsulate' function to
-                                      // take of all those possible
-                                      // ones if we simply drop in
-                                      // the address of an overloaded
-                                      // template member function,
-                                      // make it simpler for the
-                                      // compiler by giving it the
-                                      // correct type right away:
-      typedef
-       void (SparseMatrix<number>::*mem_fun_p)
-       (OutVector &,
-        const InVector &,
-        const unsigned int        ,
-        const unsigned int) const;
-      const mem_fun_p comp
-       = (&SparseMatrix<number>::
-           template threaded_vmult<OutVector,InVector>);
-      Threads::ThreadGroup<> threads;
-      for (unsigned int i=0; i<n_threads; ++i)
-       threads += Threads::spawn<void, SparseMatrix<number> > (*this, comp) (dst, src,
-                                                 n_rows * i / n_threads,
-                                                 n_rows * (i+1) / n_threads);
-      threads.join_all();
 
-      return;
-    }
-  else
-    {
-                                      // if not in MT mode or size<2000
-                                      // do it in an oldfashioned way
-      const number       *val_ptr    = &val[cols->rowstart[0]];
-      const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[0]];
-      typename OutVector::iterator dst_ptr = dst.begin();
-      for (unsigned int row=0; row<n_rows; ++row)
-       {
-         typename OutVector::value_type s = 0.;
-         const number *const val_end_of_row = &val[cols->rowstart[row+1]];
-         while (val_ptr != val_end_of_row)
-           s += *val_ptr++ * src(*colnum_ptr++);
-         *dst_ptr++ = s;
-       };
-    };
-}
-
-
-
-template <typename number>
-template <class OutVector, class InVector>
-void
-SparseMatrix<number>::threaded_vmult (OutVector       &dst,
-                                     const InVector &src,
-                                     const unsigned int        begin_row,
-                                     const unsigned int        end_row) const
-{
-                                  // this function should not be called
-                                  // when not in parallel mode.
-  Assert (DEAL_II_USE_MT, ExcInternalError());
-
-  const number       *val_ptr    = &val[cols->rowstart[begin_row]];
-  const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[begin_row]];
-  typename OutVector::iterator dst_ptr = dst.begin() + begin_row;
-  for (unsigned int row=begin_row; row<end_row; ++row)
-    {
-      typename OutVector::value_type s = 0.;
-      const number *const val_end_of_row = &val[cols->rowstart[row+1]];
-      while (val_ptr != val_end_of_row)
-       s += *val_ptr++ * src(*colnum_ptr++);
-      *dst_ptr++ = s;
-    };
+  parallel::apply_to_subranges (0U, m(),
+                               std_cxx1x::bind (internal::SparseMatrix::vmult_on_subrange
+                                                <number,InVector,OutVector>,
+                                                _1, _2,
+                                                val,
+                                                cols->rowstart,
+                                                cols->colnums,
+                                                std_cxx1x::cref(src),
+                                                std_cxx1x::ref(dst),
+                                                false),
+                               internal::SparseMatrix::minimum_parallel_grain_size);
 }
 
 
@@ -674,18 +654,17 @@ SparseMatrix<number>::vmult_add (OutVector& dst,
 
   Assert (!PointerComparison::equal(&src, &dst), ExcSourceEqualsDestination());
 
-  const unsigned int  n_rows     = m();
-  const number       *val_ptr    = &val[cols->rowstart[0]];
-  const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[0]];
-  typename OutVector::iterator dst_ptr    = dst.begin();
-  for (unsigned int row=0; row<n_rows; ++row)
-    {
-      typename OutVector::value_type s = *dst_ptr;
-      const number *const val_end_of_row = &val[cols->rowstart[row+1]];
-      while (val_ptr != val_end_of_row)
-       s += *val_ptr++ * src(*colnum_ptr++);
-      *dst_ptr++ = s;
-    };
+  parallel::apply_to_subranges (0U, m(),
+                               std_cxx1x::bind (internal::SparseMatrix::vmult_on_subrange
+                                                <number,InVector,OutVector>,
+                                                _1, _2,
+                                                val,
+                                                cols->rowstart,
+                                                cols->colnums,
+                                                std_cxx1x::cref(src),
+                                                std_cxx1x::ref(dst),
+                                                true),
+                               internal::SparseMatrix::minimum_parallel_grain_size);
 }
 
 
@@ -712,6 +691,45 @@ SparseMatrix<number>::Tvmult_add (OutVector& dst,
 }
 
 
+namespace internal
+{
+  namespace SparseMatrix
+  {
+                                    /**
+                                     * Perform a vmult using the SparseMatrix
+                                     * data structures, but only using a
+                                     * subinterval for the row indices.
+                                     *
+                                     * In the sequential case, this function
+                                     * is called on all rows, in the parallel
+                                     * case it may be called on a subrange,
+                                     * at the discretion of the task
+                                     * scheduler.
+                                     */
+    template <typename number,
+             typename InVector>
+    number matrix_norm_sqr_on_subrange (const unsigned int  begin_row,
+                                       const unsigned int  end_row,
+                                       const number       *values,
+                                       const std::size_t  *rowstart,
+                                       const unsigned int *colnums,
+                                       const InVector     &v)
+    {
+      number norm_sqr=0.;
+  
+      for (unsigned int i=begin_row; i<end_row; ++i)
+       {
+         number s = 0;
+         for (unsigned int j=rowstart[i]; j<rowstart[i+1] ;j++)
+           s += values[j] * v(colnums[j]);
+         norm_sqr += v(i)*numbers::NumberTraits<number>::conjugate(s);
+       }
+      return norm_sqr;
+    }
+  }
+}
+
+
 
 template <typename number>
 template <typename somenumber>
@@ -723,104 +741,56 @@ SparseMatrix<number>::matrix_norm_square (const Vector<somenumber>& v) const
   Assert(m() == v.size(), ExcDimensionMismatch(m(),v.size()));
   Assert(n() == v.size(), ExcDimensionMismatch(n(),v.size()));
 
-  const unsigned int n_rows = m();
-
-                                  // if in MT mode and size sufficiently
-                                  // large: do it in parallel; the limit
-                                  // is mostly artificial
-  if (DEAL_II_USE_MT &&
-      (multithread_info.n_default_threads > 1) &&
-      (n_rows/multithread_info.n_default_threads > 2000))
-    {
-      const unsigned int n_threads = multithread_info.n_default_threads;
-
-                                      // space for the norms of
-                                      // the different parts
-      std::vector<somenumber> partial_sums (n_threads, 0);
-                                      // then spawn threads. since
-                                      // some compilers have trouble
-                                      // finding out which
-                                      // 'encapsulate' function to
-                                      // take of all those possible
-                                      // ones if we simply drop in
-                                      // the address of an overloaded
-                                      // template member function,
-                                      // make it simpler for the
-                                      // compiler by giving it the
-                                      // correct type right away:
-      typedef
-       void (SparseMatrix<number>::*mem_fun_p)
-       (const Vector<somenumber> &,
-        const unsigned int        ,
-        const unsigned int        ,
-        somenumber               *) const;
-      const mem_fun_p comp
-       = (&SparseMatrix<number>::
-           template threaded_matrix_norm_square<somenumber>);
-      Threads::ThreadGroup<> threads;
-      for (unsigned int i=0; i<n_threads; ++i)
-       threads += Threads::spawn<void, SparseMatrix<number> > (*this, comp)(v,
-                                                n_rows * i / n_threads,
-                                                n_rows * (i+1) / n_threads,
-                                                &partial_sums[i]);
-
-                                      // ... and wait until they're finished
-      threads.join_all ();
-                                      // accumulate the partial results
-      return std::accumulate (partial_sums.begin(),
-                             partial_sums.end(),
-                             static_cast<somenumber>(0.));
-    }
-  else
-    {
-                                      // if not in MT mode or the matrix is
-                                      // too small: do it one-by-one
-      somenumber          sum        = 0.;
-      const number       *val_ptr    = &val[cols->rowstart[0]];
-      const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[0]];
-      for (unsigned int row=0; row<n_rows; ++row)
-       {
-         somenumber s = 0.;
-         const number *val_end_of_row = &val[cols->rowstart[row+1]];
-         while (val_ptr != val_end_of_row)
-           s += *val_ptr++ * v(*colnum_ptr++);
-         
-         sum += v(row) * numbers::NumberTraits<somenumber>::conjugate(s);
-       }
-      
-      return sum;
-    }
+  return
+    parallel::accumulate_from_subranges<number>
+    (std_cxx1x::bind (internal::SparseMatrix::matrix_norm_sqr_on_subrange
+                     <number,Vector<somenumber> >,
+                     _1, _2,
+                     val, cols->rowstart, cols->colnums,
+                     std_cxx1x::cref(v)),
+     0, m(),
+     internal::SparseMatrix::minimum_parallel_grain_size);
 }
 
 
 
-template <typename number>
-template <typename somenumber>
-void
-SparseMatrix<number>::
-threaded_matrix_norm_square (const Vector<somenumber> &v,
-                             const unsigned int        begin_row,
-                             const unsigned int        end_row,
-                             somenumber               *partial_sum) const
+namespace internal
 {
-                                  // this function should not be called
-                                  // when not in parallel mode.
-  Assert (DEAL_II_USE_MT, ExcInternalError());
-
-  somenumber sum = 0.;
-  const number       *val_ptr    = &val[cols->rowstart[begin_row]];
-  const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[begin_row]];
-  for (unsigned int row=begin_row; row<end_row; ++row)
+  namespace SparseMatrix
+  {
+                                    /**
+                                     * Perform a vmult using the SparseMatrix
+                                     * data structures, but only using a
+                                     * subinterval for the row indices.
+                                     *
+                                     * In the sequential case, this function
+                                     * is called on all rows, in the parallel
+                                     * case it may be called on a subrange,
+                                     * at the discretion of the task
+                                     * scheduler.
+                                     */
+    template <typename number,
+             typename InVector>
+    number matrix_scalar_product_on_subrange (const unsigned int  begin_row,
+                                             const unsigned int  end_row,
+                                             const number       *values,
+                                             const std::size_t  *rowstart,
+                                             const unsigned int *colnums,
+                                             const InVector     &u,
+                                             const InVector     &v)
     {
-      somenumber s = 0.;
-      const number *val_end_of_row = &val[cols->rowstart[row+1]];
-      while (val_ptr != val_end_of_row)
-       s += *val_ptr++ * v(*colnum_ptr++);
-
-      sum += v(row) * numbers::NumberTraits<somenumber>::conjugate(s);
-    }
+      number norm_sqr=0.;
   
-  *partial_sum = sum;
+      for (unsigned int i=begin_row; i<end_row; ++i)
+       {
+         number s = 0;
+         for (unsigned int j=rowstart[i]; j<rowstart[i+1] ;j++)
+           s += values[j] * v(colnums[j]);
+         norm_sqr += u(i)*numbers::NumberTraits<number>::conjugate(s);
+       }
+      return norm_sqr;
+    }
+  }
 }
 
 
@@ -836,106 +806,16 @@ SparseMatrix<number>::matrix_scalar_product (const Vector<somenumber>& u,
   Assert(m() == u.size(), ExcDimensionMismatch(m(),u.size()));
   Assert(n() == v.size(), ExcDimensionMismatch(n(),v.size()));
 
-  const unsigned int n_rows = m();
-
-                                  // if in MT mode and size sufficiently
-                                  // large: do it in parallel; the limit
-                                  // is mostly artificial
-  if (DEAL_II_USE_MT &&
-      (multithread_info.n_default_threads != 1) &&
-      (n_rows/multithread_info.n_default_threads > 2000))
-    {
-      const unsigned int n_threads = multithread_info.n_default_threads;
-
-                                      // space for the norms of
-                                      // the different parts
-      std::vector<somenumber> partial_sums (n_threads, 0);
-                                      // then spawn threads. since
-                                      // some compilers have trouble
-                                      // finding out which
-                                      // 'encapsulate' function to
-                                      // take of all those possible
-                                      // ones if we simply drop in
-                                      // the address of an overloaded
-                                      // template member function,
-                                      // make it simpler for the
-                                      // compiler by giving it the
-                                      // correct type right away:
-      typedef
-       void (SparseMatrix<number>::*mem_fun_p)
-       (const Vector<somenumber> &,
-        const Vector<somenumber> &,
-        const unsigned int        ,
-        const unsigned int        ,
-        somenumber               *) const;
-      const mem_fun_p comp
-       = (&SparseMatrix<number>::
-           template threaded_matrix_scalar_product<somenumber>);
-      Threads::ThreadGroup<> threads;
-      for (unsigned int i=0; i<n_threads; ++i)
-       threads += Threads::spawn<void, SparseMatrix<number> > (*this, comp)(u, v,
-                                                n_rows * i / n_threads,
-                                                n_rows * (i+1) / n_threads,
-                                                &partial_sums[i]);
-  
-                                      // ... and wait until they're finished
-      threads.join_all ();
-                                      // accumulate the partial results
-      return std::accumulate (partial_sums.begin(),
-                             partial_sums.end(),
-                             static_cast<somenumber>(0.));
-    }
-  else
-    {
-                                      // if not in MT mode or the matrix is
-                                      // too small: do it one-by-one
-      somenumber          sum        = 0.;
-      const number       *val_ptr    = &val[cols->rowstart[0]];
-      const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[0]];
-      for (unsigned int row=0; row<n_rows; ++row)
-       {
-         somenumber s = 0.;
-         const number *val_end_of_row = &val[cols->rowstart[row+1]];
-         while (val_ptr != val_end_of_row)
-           s += *val_ptr++ * v(*colnum_ptr++);
-         
-         sum += u(row) * numbers::NumberTraits<somenumber>::conjugate(s);
-       }
-      
-      return sum;
-    }
-}
-
-
-
-template <typename number>
-template <typename somenumber>
-void
-SparseMatrix<number>::
-threaded_matrix_scalar_product (const Vector<somenumber> &u,
-                                const Vector<somenumber> &v,
-                                const unsigned int        begin_row,
-                                const unsigned int        end_row,
-                                somenumber               *partial_sum) const
-{
-                                  // this function should not be called
-                                  // when not in parallel mode.
-  Assert (DEAL_II_USE_MT, ExcInternalError());
-
-  somenumber sum = 0.;
-  const number       *val_ptr    = &val[cols->rowstart[begin_row]];
-  const unsigned int *colnum_ptr = &cols->colnums[cols->rowstart[begin_row]];
-  for (unsigned int row=begin_row; row<end_row; ++row)
-    {
-      somenumber s = 0.;
-      const number *val_end_of_row = &val[cols->rowstart[row+1]];
-      while (val_ptr != val_end_of_row)
-       s += *val_ptr++ * v(*colnum_ptr++);
-
-      sum += u(row) * numbers::NumberTraits<somenumber>::conjugate(s);
-    }
-  
-  *partial_sum = sum;
+  return
+    parallel::accumulate_from_subranges<number>
+    (std_cxx1x::bind (internal::SparseMatrix::matrix_scalar_product_on_subrange
+                     <number,Vector<somenumber> >,
+                     _1, _2,
+                     val, cols->rowstart, cols->colnums,
+                     std_cxx1x::cref(u),
+                     std_cxx1x::cref(v)),
+     0, m(),
+     internal::SparseMatrix::minimum_parallel_grain_size);
 }
 
 
@@ -1276,6 +1156,49 @@ SparseMatrix<number>::frobenius_norm () const
 
 
 
+namespace internal
+{
+  namespace SparseMatrix
+  {
+                                    /**
+                                     * Perform a vmult using the SparseMatrix
+                                     * data structures, but only using a
+                                     * subinterval for the row indices.
+                                     *
+                                     * In the sequential case, this function
+                                     * is called on all rows, in the parallel
+                                     * case it may be called on a subrange,
+                                     * at the discretion of the task
+                                     * scheduler.
+                                     */
+    template <typename number,
+             typename InVector,
+             typename OutVector>
+    number residual_sqr_on_subrange (const unsigned int  begin_row,
+                                    const unsigned int  end_row,
+                                    const number       *values,
+                                    const std::size_t  *rowstart,
+                                    const unsigned int *colnums,
+                                    const InVector     &u,
+                                    const InVector     &b,
+                                    OutVector          &dst)
+    {
+      number norm_sqr=0.;
+  
+      for (unsigned int i=begin_row; i<end_row; ++i)
+       {
+         number s = b(i);
+         for (unsigned int j=rowstart[i]; j<rowstart[i+1] ;j++)
+           s -= values[j] * u(colnums[j]);
+         dst(i) = s;
+         norm_sqr += s*numbers::NumberTraits<number>::conjugate(s);
+       }
+      return norm_sqr;
+    }
+  }
+}
+
+
 template <typename number>
 template <typename somenumber>
 somenumber
@@ -1291,107 +1214,17 @@ SparseMatrix<number>::residual (Vector<somenumber>       &dst,
 
   Assert (&u != &dst, ExcSourceEqualsDestination());
 
-  const unsigned int n_rows = m();
-
-                                  // if in MT mode and size sufficiently
-                                  // large: do it in parallel; the limit
-                                  // is mostly artificial
-  if (DEAL_II_USE_MT &&
-      (multithread_info.n_default_threads > 1) &&
-      (n_rows/multithread_info.n_default_threads > 2000))
-    {
-      const unsigned int n_threads = multithread_info.n_default_threads;
-                                      // space for the square norms of
-                                      // the different parts
-      std::vector<somenumber> partial_norms (n_threads, 0);
-
-                                      // then spawn threads. since
-                                      // some compilers have trouble
-                                      // finding out which
-                                      // 'encapsulate' function to
-                                      // take of all those possible
-                                      // ones if we simply drop in
-                                      // the address of an overloaded
-                                      // template member function,
-                                      // make it simpler for the
-                                      // compiler by giving it the
-                                      // correct type right away:
-      typedef
-       void (SparseMatrix<number>::*mem_fun_p)
-       (Vector<somenumber>       &,
-        const Vector<somenumber> &,
-        const Vector<somenumber> &,
-        const std::pair<unsigned int,unsigned int>,
-        somenumber               *) const;
-      const mem_fun_p comp_residual = &SparseMatrix<number>::
-                                    template threaded_residual<somenumber>;
-      Threads::ThreadGroup<> threads;
-      for (unsigned int i=0; i<n_threads; ++i)
-       threads += Threads::spawn<void, SparseMatrix<number> > (*this, comp_residual)(dst, u, b,
-                                                         std::pair<unsigned int,unsigned int>
-                                                         (n_rows * i / n_threads,
-                                                          n_rows * (i+1) / n_threads),
-                                                         &partial_norms[i]);
-
-                                      // ... and wait until they're finished
-      threads.join_all ();
-                                      // accumulate the partial results
-      return std::sqrt(std::accumulate (partial_norms.begin(),
-                                       partial_norms.end(),
-                                       static_cast<somenumber>(0.)));
-    }
-  else
-    {
-      somenumber norm=0.;   
-  
-      for (unsigned int i=0; i<n_rows; ++i)
-       {
-         somenumber s = b(i);
-         for (unsigned int j=cols->rowstart[i]; j<cols->rowstart[i+1] ;j++)
-           {
-             const unsigned int p = cols->colnums[j];
-             s -= val[j] * u(p);
-           }
-         dst(i) = s;
-         norm += dst(i)*dst(i);
-       }
-      return std::sqrt(norm);
-    };
-}
-
-
-template <typename number>
-template <typename somenumber>
-void
-SparseMatrix<number>::threaded_residual (Vector<somenumber>       &dst,
-                                        const Vector<somenumber> &u,
-                                        const Vector<somenumber> &b,
-                                        const std::pair<unsigned int, unsigned int> interval,
-                                        somenumber               *partial_norm) const
-{
-                                  // this function should not be called
-                                  // when not in parallel mode.
-  Assert (DEAL_II_USE_MT, ExcInternalError());
-
-  const unsigned int begin_row = interval.first,
-                    end_row   = interval.second;
-  
-  somenumber norm=0.;   
-  
-  for (unsigned int i=begin_row; i<end_row; ++i)
-    {
-      somenumber s = b(i);
-      for (unsigned int j=cols->rowstart[i]; j<cols->rowstart[i+1] ;j++)
-       {
-         const unsigned int p = cols->colnums[j];
-         s -= val[j] * u(p);
-       }
-      dst(i) = s;
-      norm += dst(i)*dst(i);
-    };
-
-  *partial_norm = norm;
+  return
+    std::sqrt (parallel::accumulate_from_subranges<number>
+              (std_cxx1x::bind (internal::SparseMatrix::residual_sqr_on_subrange
+                                <number,Vector<somenumber>,Vector<somenumber> >,
+                                _1, _2,
+                                val, cols->rowstart, cols->colnums,
+                                std_cxx1x::cref(u),
+                                std_cxx1x::cref(b),
+                                std_cxx1x::ref(dst)),
+               0, m(),
+               internal::SparseMatrix::minimum_parallel_grain_size));
 }
 
 
index 1a80a04be15af02b1b15a51f779d1a0bc64d4445..a0cd767ffa82f6e4738c1c52fc07215ea56df8a2 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -118,8 +118,9 @@ SparseVanka<number>::compute_inverses ()
                                        // Now spawn the threads
       Threads::ThreadGroup<> threads;
       for (unsigned int i=0; i<n_threads; ++i)
-        threads += Threads::spawn (*this, fun_ptr)(blocking[i].first,
-                                                   blocking[i].second);
+        threads += Threads::new_thread (fun_ptr, *this,
+                                       blocking[i].first,
+                                       blocking[i].second);
       threads.join_all ();
     };
 }
@@ -604,8 +605,9 @@ void SparseBlockVanka<number>::vmult (Vector<number2>       &dst,
             = &SparseVanka<number>::template apply_preconditioner<number2>;
           Threads::ThreadGroup<> threads;
           for (unsigned int block=0; block<n_blocks; ++block)
-            threads += Threads::spawn<void,SparseVanka<number> >(*static_cast<const SparseVanka<number>*>(this), comp)
-                       (dst, src,&dof_masks[block]);
+            threads += Threads::new_thread (comp,
+                                           *static_cast<const SparseVanka<number>*>(this),
+                                           dst, src,&dof_masks[block]);
           threads.join_all ();
         }
       else
index fb5a85a07659503f25e5145d81606f9cc2029910..87ed650c4ad6797554dc5e6649cba0d8e1be31c2 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006 by the deal.II authors
+//    Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2008 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -168,7 +168,7 @@ void SwappableVector<number>::alert ()
                                     // data has not been preloaded so
                                     // far, so go on! For this, start
                                     // a detached thread
-    Threads::spawn (*this, &SwappableVector<number>::reload_vector)(true);
+    Threads::new_thread (&SwappableVector<number>::reload_vector, *this, true);
                                   // note that reload_vector also
                                   // releases the lock
 }
index cc6aa2ddcc2b58795ccaf10062d07ee58627f9c0..f3ec98636754cdf30916d133f19130557e25580f 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -51,6 +51,33 @@ template <typename> class BlockVector;
 
 template <typename> class VectorView;
 
+namespace internal
+{
+  namespace Vector
+  {
+                                    /**
+                                     * If we do computations on vectors in
+                                     * parallel (say, we add two vectors to
+                                     * get a third, and we do the loop over
+                                     * all elements in parallel), then this
+                                     * variable determines the minimum number
+                                     * of elements for which it is profitable
+                                     * to split a range of elements any
+                                     * further to distribute to different
+                                     * threads.
+                                     *
+                                     * This variable is available as a global
+                                     * writable variable in order to allow
+                                     * the testsuite to also test the
+                                     * parallel case. By default, it is set
+                                     * to several thousand elements, which is
+                                     * a case that the testsuite would not
+                                     * normally encounter.
+                                     */
+    extern unsigned int minimum_parallel_grain_size;
+  }
+}
+
 
 
 /*! @addtogroup Vectors
@@ -117,6 +144,8 @@ class Vector : public Subscriptor
                                      */
     typedef typename numbers::NumberTraits<Number>::real_type real_type;
 
+  public:
+    
                                     /**
                                      * @name 1: Basic Object-handling 
                                      */
index 20555a62b9c40cf184f81350b3b8edc47a2046e3..5916637e5715fe6d5d66c73ad5a10eafe6f95246 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
@@ -16,6 +16,7 @@
 
 #include <base/template_constraints.h>
 #include <base/numbers.h>
+#include <base/parallel.h>
 #include <lac/vector.h>
 #include <lac/block_vector.h>
 
@@ -28,6 +29,8 @@
 #  include <lac/trilinos_vector.h>
 #endif
 
+#include <boost/lambda/lambda.hpp>
+
 #include <cmath>
 #include <cstring>
 #include <algorithm>
@@ -411,8 +414,12 @@ Vector<Number>& Vector<Number>::operator -= (const Vector<Number>& v)
   Assert (vec_size!=0, ExcEmptyObject());
   Assert (vec_size == v.vec_size, ExcDimensionMismatch(vec_size, v.vec_size));
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] -= v.val[i];
+  parallel::transform (val,
+                      val+vec_size,
+                      v.val,
+                      val,
+                      (boost::lambda::_1 - boost::lambda::_2),
+                      internal::Vector::minimum_parallel_grain_size);
 
   return *this;
 }
@@ -423,8 +430,11 @@ void Vector<Number>::add (const Number v)
 {
   Assert (vec_size!=0, ExcEmptyObject());
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] += v;
+  parallel::transform (val,
+                      val+vec_size,
+                      val,
+                      (boost::lambda::_1 + v),
+                      internal::Vector::minimum_parallel_grain_size);
 }
 
 
@@ -434,8 +444,12 @@ void Vector<Number>::add (const Vector<Number>& v)
   Assert (vec_size!=0, ExcEmptyObject());
   Assert (vec_size == v.vec_size, ExcDimensionMismatch(vec_size, v.vec_size));
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] += v.val[i];
+  parallel::transform (val,
+                      val+vec_size,
+                      v.val,
+                      val,
+                      (boost::lambda::_1 + boost::lambda::_2),
+                      internal::Vector::minimum_parallel_grain_size);
 }
 
 
@@ -452,8 +466,13 @@ void Vector<Number>::add (const Number a, const Vector<Number>& v,
   Assert (vec_size == v.vec_size, ExcDimensionMismatch(vec_size, v.vec_size));
   Assert (vec_size == w.vec_size, ExcDimensionMismatch(vec_size, w.vec_size));
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] += a*v.val[i] + b*w.val[i];
+  parallel::transform (val,
+                      val+vec_size,
+                      v.val,
+                      w.val,
+                      val,
+                      (boost::lambda::_1 + a*boost::lambda::_2 + b*boost::lambda::_3),
+                      internal::Vector::minimum_parallel_grain_size);
 }
 
 
@@ -467,8 +486,12 @@ void Vector<Number>::sadd (const Number x,
   Assert (vec_size!=0, ExcEmptyObject());
   Assert (vec_size == v.vec_size, ExcDimensionMismatch(vec_size, v.vec_size));
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] = x*val[i] + v.val[i];
+  parallel::transform (val,
+                      val+vec_size,
+                      v.val,
+                      val,
+                      (x*boost::lambda::_1 + boost::lambda::_2),
+                      internal::Vector::minimum_parallel_grain_size);
 }
 
 
@@ -489,8 +512,13 @@ void Vector<Number>::sadd (const Number x, const Number a,
   Assert (vec_size == v.vec_size, ExcDimensionMismatch(vec_size, v.vec_size));
   Assert (vec_size == w.vec_size, ExcDimensionMismatch(vec_size, w.vec_size));
 
-  for (unsigned int i=0; i<vec_size; ++i)
-    val[i] = x*val[i] + a*v.val[i] + b*w.val[i];
+  parallel::transform (val,
+                      val+vec_size,
+                      v.val,
+                      w.val,
+                      val,
+                      (x*boost::lambda::_1 + a*boost::lambda::_2 + b*boost::lambda::_3),
+                      internal::Vector::minimum_parallel_grain_size);
 }
 
 
index dfd53724b7d386984f499d60de44111c013b3af7..0a703781c385e663227bbf6418f407eb441537d3 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 #include <lac/block_vector.h>
 
 DEAL_II_NAMESPACE_OPEN
+
+namespace internal
+{
+  namespace SparseMatrix
+  {
+                                    // set this value to 1/5 of the value of
+                                    // the minimum grain size of
+                                    // vectors. this rests on the fact that
+                                    // we have to do a lot more work per row
+                                    // of a matrix than per element of a
+                                    // vector. it could possibly be reduced
+                                    // even further but that doesn't appear
+                                    // worth it any more for anything but
+                                    // very small matrices that we don't care
+                                    // that much about anyway.
+    unsigned int minimum_parallel_grain_size = 200;
+  }
+}
+
+
 #include "sparse_matrix.inst"
+
 DEAL_II_NAMESPACE_CLOSE
index 2834504ba6d9775dc91c8ccaaeab53230156fa4a..f37e3c8b64a477680bc85048eab9f5ada525391a 100644 (file)
@@ -2,7 +2,7 @@
 //    $Id$
 //    Version: $Name$
 //
-//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 by the deal.II authors
+//    Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 by the deal.II authors
 //
 //    This file is subject to QPL and may not be  distributed
 //    without copyright and license information. Please refer
 
 DEAL_II_NAMESPACE_OPEN
 
+
+namespace internal
+{
+  namespace Vector
+  {
+                                    // set minimum grain size. this value is
+                                    // roughly in accordance with the curve
+                                    // in the TBB book (fig 3.2) that shows
+                                    // run time as a function of grain size
+                                    // -- there, values from 200 upward are
+                                    // so that the scheduling overhead
+                                    // amortizes well (for very large values
+                                    // in that example, the grain size is too
+                                    // large to split the work load into
+                                    // enough chunks and the problem becomes
+                                    // badly balanced)
+    unsigned int minimum_parallel_grain_size = 1000;
+  }
+}
+
+
 #include "vector.inst"
 
 // do a few functions that currently don't fit the scheme because they have
index 4a761062841f244853bb7a86537a594852a5d3c0..714f6ec82aa9b426d9f9bfbb54a6efeda6688567 100644 (file)
@@ -134,6 +134,9 @@ clean-base clean-lac clean-1d clean-2d clean-3d:
 
 clean-contrib:
        -rm contrib/*/*.$(OBJEXT)
+       -rm -r contrib/tbb/*debug
+       -rm -r contrib/tbb/*release
+       -rm libtbb*
 
 clean-bin:
        -rm bin/detached_ma27

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.