--- /dev/null
+# $Id$
+
+# include paths and global variables
+include ../../common/Make.global_options
+
+# check xrefs in all subdirs
+validate-xrefs:
+ for dir in * ; do \
+ if test -d $$dir -a ! $$dir = CVS ; then \
+ cd $$dir ; \
+ echo "Checking `pwd`" ; \
+ $(PERL) $D/doc/auto/scripts/validate-xrefs.pl *html ; \
+ cd .. ; \
+ fi ; \
+ done
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="previous" HREF="node6.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
</HEAD>
<BODY >
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node2.html">
-<LINK REL="previous" HREF="main.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="previous" HREF="index.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node2.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html42"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html36"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
SRC="previous_motif.gif"></A>
<BR>
<B> Next:</B> <A NAME="tex2html45"
HREF="node2.html">Grid handling</A>
<B> Up:</B> <A NAME="tex2html43"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html37"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<BR>
<BR>
<!--End of Navigation Panel-->
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html42"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html36"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
SRC="previous_motif.gif"></A>
<BR>
<B> Next:</B> <A NAME="tex2html45"
HREF="node2.html">Grid handling</A>
<B> Up:</B> <A NAME="tex2html43"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html37"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<!--End of Navigation Panel-->
<ADDRESS>
<I>Wolfgang Bangerth</I>
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node3.html">
<LINK REL="previous" HREF="node1.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node3.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html58"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html52"
<B> Next:</B> <A NAME="tex2html61"
HREF="node3.html">Finite element spaces</A>
<B> Up:</B> <A NAME="tex2html59"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html53"
HREF="node1.html">Design and evolution of</A>
<BR>
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html58"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html52"
<B> Next:</B> <A NAME="tex2html61"
HREF="node3.html">Finite element spaces</A>
<B> Up:</B> <A NAME="tex2html59"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html53"
HREF="node1.html">Design and evolution of</A>
<!--End of Navigation Panel-->
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node4.html">
<LINK REL="previous" HREF="node2.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node4.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html70"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html64"
<B> Next:</B> <A NAME="tex2html73"
HREF="node4.html">Iterative solvers</A>
<B> Up:</B> <A NAME="tex2html71"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html65"
HREF="node2.html">Grid handling</A>
<BR>
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html70"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html64"
<B> Next:</B> <A NAME="tex2html73"
HREF="node4.html">Iterative solvers</A>
<B> Up:</B> <A NAME="tex2html71"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html65"
HREF="node2.html">Grid handling</A>
<!--End of Navigation Panel-->
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node5.html">
<LINK REL="previous" HREF="node3.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node5.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html83"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html77"
<B> Next:</B> <A NAME="tex2html86"
HREF="node5.html">Example applications</A>
<B> Up:</B> <A NAME="tex2html84"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html78"
HREF="node3.html">Finite element spaces</A>
<BR>
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html83"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html77"
<B> Next:</B> <A NAME="tex2html86"
HREF="node5.html">Example applications</A>
<B> Up:</B> <A NAME="tex2html84"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html78"
HREF="node3.html">Finite element spaces</A>
<!--End of Navigation Panel-->
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node6.html">
<LINK REL="previous" HREF="node4.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node6.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html96"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html90"
<B> Next:</B> <A NAME="tex2html99"
HREF="node6.html">Bibliography</A>
<B> Up:</B> <A NAME="tex2html97"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html91"
HREF="node4.html">Iterative solvers</A>
<BR>
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html96"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html90"
<B> Next:</B> <A NAME="tex2html99"
HREF="node6.html">Bibliography</A>
<B> Up:</B> <A NAME="tex2html97"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html91"
HREF="node4.html">Iterative solvers</A>
<!--End of Navigation Panel-->
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="next" HREF="node7.html">
<LINK REL="previous" HREF="node5.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
<LINK REL="next" HREF="node7.html">
</HEAD>
<BODY >
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif.gif"></A>
<A NAME="tex2html110"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html104"
<B> Next:</B> <A NAME="tex2html113"
HREF="node7.html">About this document ...</A>
<B> Up:</B> <A NAME="tex2html111"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html105"
HREF="node5.html">Example applications</A>
<BR><BR>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
<LINK REL="STYLESHEET" HREF="screen.css" media="screen">
<LINK REL="previous" HREF="node6.html">
-<LINK REL="up" HREF="main.html">
+<LINK REL="up" HREF="index.html">
</HEAD>
<BODY >
<!--Navigation Panel-->
<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
SRC="next_motif_gr.gif">
<A NAME="tex2html118"
- HREF="main.html">
+ HREF="index.html">
<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
SRC="up_motif.gif"></A>
<A NAME="tex2html114"
SRC="previous_motif.gif"></A>
<BR>
<B> Up:</B> <A NAME="tex2html119"
- HREF="main.html">No Title</A>
+ HREF="index.html">No Title</A>
<B> Previous:</B> <A NAME="tex2html115"
HREF="node6.html">Bibliography</A>
<BR>
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN"
- "http://www.w3.org/TR/REC-html40/strict.dtd">
-<html>
- <head>
- <link href="../../screen.css" rel="StyleSheet" title="deal.II Homepage" media="screen">
- <link href="../../print.css" rel="StyleSheet" title="deal.II Homepage" media="print">
- <title>The deal.II Homepage</title>
- <meta name="author" content="Wolfgang Bangerth <deal@iwr.uni-heidelberg.de>">
- <meta name="keywords" content="deal.II"></head>
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>No Title</TITLE>
+<META NAME="description" CONTENT="No Title">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node1.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html1"
+ HREF="node1.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif_gr.gif">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif_gr.gif">
+<BR>
+<B> Next:</B> <A NAME="tex2html2"
+ HREF="node1.html">Background</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
- </head>
+<P>
+<DIV ALIGN="CENTER">
+<FONT SIZE="+3">Multi-threading support in <TT>deal.II</TT>
+ </FONT>
+<P>
+<BR>
+<BR>
-<body>
+<P>
+<FONT SIZE="+1">Wolfgang Bangerth
+<BR>
+University of Heidelberg
+<BR>
+<BR>
+March 2000
+ </FONT></DIV>
+<P>
-<center>
- <h2>Multi-threading support in deal.II</h2>
- <h3>Wolfgang Bangerth, March 2000</h3>
-</center>
-
-<DIR>
- In this report, we describe the implementational techniques of
- multi-threading support in deal.II, which we use for the
+<H3>Abstract:</H3>
+<DIV>
+In this report, we describe the implementational techniques of
+ multi-threading support in <TT>deal.II</TT>, which we use for the
parallelization of independent operations. Writing threaded programs in
- C++ is obstructed by two problems: operating system dependent
- interfaces and that these interfaces are created for C programs
- rather than for C++. We present our solutions to these problems and
- describe first experiences using multi-threading in deal.II.
-</DIR>
-
-<strong>
-Note: this report is presently pending acceptance as Preprint at the
-IWR. The expected publication date is end of April/beginning of May,
-so you may want to check out the
-<a href="http://www.iwr.uni-heidelberg.de" target="_top">IWR preprint
-server</a> or the <a href="http://gaia.iwr.uni-heidelberg.de/~deal"
-target="_top">deal.II homepage</a> after that date.
-</strong>
-
-</body>
-</html>
-
+ <TT>C++</TT> is obstructed by two problems: operating system dependent
+ interfaces and that these interfaces are created for <TT>C</TT> programs
+ rather than for <TT>C++</TT>. We present our solutions to these problems and
+ describe first experiences using multi-threading in <TT>deal.II</TT>.
+</DIV>
+<P>
+<P>
+<BR><HR>
+<!--Table of Child-Links-->
+<A NAME="CHILD_LINKS"> </A>
+<UL>
+<LI><A NAME="tex2html3"
+ HREF="node1.html">Background</A>
+<LI><A NAME="tex2html4"
+ HREF="node2.html">Threads</A>
+<LI><A NAME="tex2html5"
+ HREF="node3.html">Creating and managing threads</A>
+<UL>
+<LI><A NAME="tex2html6"
+ HREF="node4.html">Operating system dependence and ACE</A>
+<LI><A NAME="tex2html7"
+ HREF="node5.html"><TT>C</TT> interface to threads versus <TT>C++</TT></A>
+<LI><A NAME="tex2html8"
+ HREF="node6.html">First approach</A>
+<LI><A NAME="tex2html9"
+ HREF="node7.html">Second approach</A>
+<UL>
+<LI><A NAME="tex2html10"
+ HREF="node7.html#SECTION00034100000000000000">Finding the correct template parameters.</A>
+<LI><A NAME="tex2html11"
+ HREF="node7.html#SECTION00034200000000000000">Encapsulating the parameters.</A>
+<LI><A NAME="tex2html12"
+ HREF="node7.html#SECTION00034300000000000000">Using these objects.</A>
+<LI><A NAME="tex2html13"
+ HREF="node7.html#SECTION00034400000000000000">Virtual constructors.</A>
+<LI><A NAME="tex2html14"
+ HREF="node7.html#SECTION00034500000000000000">Spawning independent threads.</A>
+<LI><A NAME="tex2html15"
+ HREF="node7.html#SECTION00034600000000000000">Number of parameters. Non-member functions.</A>
+</UL>
+</UL>
+<LI><A NAME="tex2html16"
+ HREF="node8.html">Applications</A>
+<UL>
+<LI><A NAME="tex2html17"
+ HREF="node9.html">Writing output detached to disk</A>
+<LI><A NAME="tex2html18"
+ HREF="node10.html">Assembling the matrix</A>
+<LI><A NAME="tex2html19"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+</UL>
+<LI><A NAME="tex2html20"
+ HREF="node12.html">Conclusions</A>
+<UL>
+<LI><A NAME="tex2html21"
+ HREF="node12.html#SECTION00050010000000000000">Acknowledgments.</A>
+</UL>
+<LI><A NAME="tex2html22"
+ HREF="node13.html">Bibliography</A>
+<LI><A NAME="tex2html23"
+ HREF="node14.html">About this document ... </A>
+</UL>
+<!--End of Table of Child-Links-->
+<BR><HR>
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+
+body { background-image: none;
+ background-color: white;
+ }
+
+frameset { background-color: white;
+ border-color: white;
+ border: 0;
+ frameborder: 0;
+ }
+
+td { vertical-align: top; }
+
+dt { font-weight: bold; }
+
+.figure { font-weight: bold;
+ font-size: larger;
+ }
+
+.pagetoc {}
+
+.chapter_title {}
+
+span.parhead { font-weight: bold;
+ }
+
+span.example { font-weight: bold;
+ font-style: italic;
+ }
+pre.example { padding-left: 2 em;
+ text-align: left;
+ text-indent: 0;
+ }
+
+table.navbar { }
+
+
+
+/* Century Schoolbook font is very similar to Computer Modern Math: cmmi */
+.MATH { font-family: "Century Schoolbook", serif; }
+.MATH I { font-family: "Century Schoolbook", serif; font-weight: bold }
+.BOLDMATH { font-family: "Century Schoolbook", serif; font-weight: bold }
+
+/* implement both fixed-size and relative sizes */
+SMALL.XTINY { font-size : xx-small }
+SMALL.TINY { font-size : x-small }
+SMALL.SCRIPTSIZE { font-size : smaller }
+SMALL.FOOTNOTESIZE { font-size : small }
+SMALL.SMALL { }
+BIG.LARGE { }
+BIG.XLARGE { font-size : large }
+BIG.XXLARGE { font-size : x-large }
+BIG.HUGE { font-size : larger }
+BIG.XHUGE { font-size : xx-large }
+
+/* document-specific styles come next */
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Background</TITLE>
+<META NAME="description" CONTENT="Background">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node2.html">
+<LINK REL="previous" HREF="index.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node2.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html32"
+ HREF="node2.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html30"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html24"
+ HREF="index.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html33"
+ HREF="node2.html">Threads</A>
+<B> Up:</B> <A NAME="tex2html31"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html25"
+ HREF="index.html">No Title</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00010000000000000000">
+Background</A>
+</H1>
+
+<P>
+Realistic finite element simulations tend to use enormous amounts of computing
+time and memory. Scientists and programmers have therefore long tried to use
+the combined power of several processors or computers to tackle these
+problems.
+
+<P>
+The usual approach is to use physically separated computers (e.g. clusters) or
+computing units (e.g. processor nodes in a parallel computer), each of which
+is equipped with its own memory, and split the problem at hand into separate
+parts which are then solved on these computing units. Unfortunately, this
+approach tends to pose significant problems, both for the mathematical
+formulation as well as for the application programmer, which make the
+development of such programs overly difficult and expensive.
+
+<P>
+For these reasons, parallelized implementations and their
+mathematical background are still subject to intense research. In recent
+years, however, multi-processor machines have been developed, which pose a
+reasonable alternative to small parallel computers with the advantage of
+simple programming and the possibility to use the same mathematical
+formulation that can also be used for single-processor machines. These
+computers typically have between two and eight processors that can access the
+global memory at equal cost.
+
+<P>
+Due to this uniform memory access (UMA) architecture, communication can be
+performed in the global memory and is no more costly than access to any other
+memory location. Thus, there is also no more need to change the mathematical
+formulation to reduce communication, and programs using this architecture look
+very much like programs written for single processor machines.
+
+<P>
+The purpose of this report is to explain the techniques used in
+<TT>deal.II</TT> (see [<A
+ HREF="node13.html#BK99tr">1</A>,<A
+ HREF="node13.html#BK99m">2</A>])
+by which we try to program these computers. We will first
+give a brief introduction in what threads are and what the problems are which
+we have to solve when we want to use multi-threading. The third section takes an
+in-depth look at the way in which the functionality of the operating system is
+represented in a <TT>C++</TT> program in order to allow simple and robust
+programming; in particular, we describe the design decisions which led us to
+implement these parts of the library in the way they are implemented. In the
+fourth section, we show several examples of parallelization and explain how
+they work. Readers who are more interested in actually using the framework
+laid out in this report, rather than the internals, may skip Section 3 and go
+directly to the applications in Section <A HREF="node8.html#sec:applications">4</A> (page
+<A HREF="node8.html#sec:applications"><IMG ALIGN="BOTTOM" BORDER="1" ALT="[*]"
+ SRC="../deal.II-paper/cross_ref_motif.gif"></A>).
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html32"
+ HREF="node2.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html30"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html24"
+ HREF="index.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html33"
+ HREF="node2.html">Threads</A>
+<B> Up:</B> <A NAME="tex2html31"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html25"
+ HREF="index.html">No Title</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Assembling the matrix</TITLE>
+<META NAME="description" CONTENT="Assembling the matrix">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node11.html">
+<LINK REL="previous" HREF="node9.html">
+<LINK REL="up" HREF="node8.html">
+<LINK REL="next" HREF="node11.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html139"
+ HREF="node11.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html137"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html131"
+ HREF="node9.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html140"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+<B> Up:</B> <A NAME="tex2html138"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html132"
+ HREF="node9.html">Writing output detached to</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00042000000000000000">
+Assembling the matrix</A>
+</H2>
+
+<P>
+Setting up the system matrix is usually done by looping over all cells and
+computing the contributions of each cell separately. While the computations of
+the local contributions is strictly independent, we need to transfer these
+contributions to the global matrix afterward. This transfer has to be
+synchronized, in order to avoid that one thread overwrites values that another
+thread has just written.
+
+<P>
+In most cases, building the system matrix in parallel will look like the
+following template:
+<PRE>
+ void MainClass::build_matrix () {
+ // define how many threads will be used (here: 4)
+ const unsigned int n_threads = 4;
+ const unsigned int n_cells_per_thread
+ = triangulation.n_active_cells () / n_threads;
+
+ // define the Mutex that will be used to synchronise
+ // accesses to the matrix
+ ACE_Thread_Mutex mutex;
+
+ // define thread manager
+ ACE_Thread_Manager thread_manager;
+
+ vector<DoFHandler<dim>::active_cell_iterator>
+ first_cells (n_threads),
+ end_cells (n_threads);
+
+ DoFHandler<dim>::active_cell_iterator
+ present_cell = dof_handler.begin_active ();
+ for (unsigned int thread=0; thread<n_threads; ++thread)
+ {
+ // for each thread: first determine the range of cells on
+ // which it shall operate:
+ first_cells[thread] = present_cell;
+
+ end_cells[thread] = first_cells[thread];
+ if (thread != n_threads-1)
+ for (unsigned int i=0; i<n_cells_per_thread; ++i)
+ ++end_cells[thread];
+ else
+ end_cells[thread] = dof_handler.end();
+
+ // now start a new thread that builds the contributions of
+ // the cells in the given range
+ Threads::spawn (thread_manager,
+ Threads::encapsulate(&MainClass::build_matrix_threaded)
+ .collect_args (this,
+ first_cells[thread],
+ end_cells[thread],
+ mutex));
+
+ // set start iterator for next thread
+ present_cell = end_cells[thread];
+ };
+
+ // wait for the threads to finish
+ thread_manager.wait ();
+ };
+
+
+ void MainClass::build_matrix_threaded
+ (const DoFHandler<dim>::active_cell_iterator &first_cell,
+ const DoFHandler<dim>::active_cell_iterator &end_cell,
+ ACE_Thread_Mutex &mutex)
+ {
+ FullMatrix<double> cell_matrix;
+ vector<unsigned int> local_dof_indices;
+
+ DoFHandler<dim>::active_cell_iterator cell;
+ for (cell=first_cell; cell!=end_cell; ++cell)
+ {
+ // compute the elements of the cell matrix
+ ...
+
+ // get the indices of the DoFs of this cell
+ cell->get_dof_indices (local_dof_indices);
+
+ // now transfer local matrix into the global one.
+ // synchronise this with the other threads
+ mutex.acquire ();
+ for (unsigned int i=0; i<dofs_per_cell; ++i)
+ for (unsigned int j=0; j<dofs_per_cell; ++j)
+ global_matrix.add (local_dof_indices[i],
+ local_dof_indices[j],
+ cell_matrix(i,j));
+ mutex.release ();
+ };
+ };
+</PRE>
+<P>
+Note that since the <TT>build_matrix_threaded</TT> function takes its
+arguments as references, we have to make sure that the variables to which
+these references point live at least as long as the spawned threads. It is
+thus not possible to use the same variables for start and end iterator for all
+threads, as the following example would do:
+<PRE>
+ ....
+ DoFHandler<dim>::active_cell_iterator
+ first_cell = dof_handler.begin_active ();
+ for (unsigned int thread=0; thread<n_threads; ++thread)
+ {
+ // for each thread: first determine the range of threads on
+ // which it shall operate:
+ DoFHandler<dim>::active_cell_iterator end_cell = first_cell;
+ if (thread != n_threads-1)
+ for (unsigned int i=0; i<n_cells_per_thread; ++i)
+ ++end_cell;
+ else
+ end_cell = dof_handler.end();
+
+ // now start a new thread that builds the contributions of
+ // the cells in the given range
+ Threads::spawn (thread_manager,
+ Threads::encapsulate(&MainClass::build_matrix_threaded)
+ .collect_args (this, first_cell, end_cell, mutex));
+
+ // set start iterator for next thread
+ first_cell = end_cell;
+ };
+ ....
+</PRE>
+<P>
+Since splitting a range of iterators (for example the range
+<TT>begin_active()</TT> to <TT>end()</TT>) is a very common task when setting
+up threads, there is a function
+<PRE>
+ template <typename ForwardIterator>
+ vector<pair<ForwardIterator,ForwardIterator> >
+ split_range (const ForwardIterator &begin, const ForwardIterator &end,
+ const unsigned int n_intervals);
+</PRE>in the <TT>Threads</TT> namespace that splits the range
+[<TT>begin</TT>,<TT>end</TT>) into <TT>n_intervals</TT> subintervals of
+approximately the same size.
+
+<P>
+Using this function, the thread creation function can now be written as
+follows:
+<PRE>
+ void MainClass::build_matrix () {
+ const unsigned int n_threads = 4;
+ ACE_Thread_Mutex mutex;
+ ACE_Thread_Manager thread_manager;
+
+ // define starting and end point for each thread
+ typedef DoFHandler<dim>::active_cell_iterator active_cell_iterator;
+ vector<pair<active_cell_iterator,active_cell_iterator> >
+ thread_ranges
+ = split_range<active_cell_iterator> (dof_handler.begin_active (),
+ dof_handler.end (),
+ n_threads);
+
+ for (unsigned int thread=0; thread<n_threads; ++thread)
+ spawn (thread_manager,
+ encapsulate(&MainClass::build_matrix_threaded)
+ .collect_args (this,
+ thread_ranges[thread].first,
+ thread_ranges[thread].second,
+ mutex));
+
+ thread_manager.wait ();
+ };
+</PRE>We have here omitted the <TT>Threads::</TT> prefix to make things more
+readable. Note that we had to explicitly specify the iterator type
+<TT>active_cell_iterator</TT> to the <TT>split_range</TT> function, since
+the two iterators given have different type (<TT>dof_handler.end()</TT> has
+type <TT>DoFHandler<dim> :: raw_cell_iterator</TT>, which can be converted to
+<TT>DoFHandler<dim>::active_cell_iterator</TT>) and C++ requires that either
+the type is explicitly given or the type be unique.
+
+<P>
+A word of caution is in place here: since usually in finite element
+computations, the system matrix is ill-conditioned, small changes in a data
+vector or the matrix can lead to significant changes in the output.
+Unfortunately, since the order in which contributions to elements of the
+matrix or vector are computed can not be predicted when using multiple
+threads, round-off can come into play here. For example, taken from a
+real-world program, the following contributions for an element of a right hand
+side vector are computed from four cells:
+<!-- MATH: $-3.255208333333328815$ -->
+-3.255208333333328815,
+
+<!-- MATH: $-3.255208333333333694$ -->
+-3.255208333333333694,
+<!-- MATH: $-3.255208333333333694$ -->
+-3.255208333333333694, and
+<!-- MATH: $-3.255208333333331526$ -->
+-3.255208333333331526;
+however, due to round-off the sum of these numbers depends on the order in
+which they are summed up, such that the resulting element of the vector
+differed depending on the number of threads used, the number of other programs
+on the computer, and other random sources. In subsequent runs of exactly the
+same programs, the sum was either
+<!-- MATH: $-13.02083333333332827$ -->
+-13.02083333333332827 or
+
+<!-- MATH: $-13.02083333333332610$ -->
+-13.02083333333332610. Although the difference is still only in the range
+of round-off error, it caused a change in the fourth digit of a derived, very
+ill-conditioned quantity after the matrix was inverted several times (this
+accuracy in this quantity was not really needed, but it showed up in the
+output and also led to different grid refinement due to comparison with other
+values of almost the same size). Tracking down the source of such problems is
+extremely difficult and frustrating, since they occur non-deterministically in
+subsequent runs of the same program, and it can take several days until the
+actual cause is found.
+
+<P>
+One possible work-around is to reduce the accuracy of the summands such that
+the value of the sum becomes irrespective of the order of the summands. One,
+rather crude method is to use a conversion to data type <TT>float</TT> and
+back; the update loop from above would then look as follows:
+<PRE>
+ for (unsigned int i=0; i<dofs_per_cell; ++i)
+ for (unsigned int j=0; j<dofs_per_cell; ++j)
+ global_matrix.add (local_dof_indices[i],
+ local_dof_indices[j],
+ static_cast<float>(cell_matrix(i,j)));
+</PRE>Note that the cast back to <TT>double</TT> is performed here implicitly. The
+question whether a reduction in accuracy in the order shown here is tolerable,
+is problem dependent. There are methods that lose less accuracy than shown
+above.
+
+<P>
+The other, less computationally costly possibility would be to decrease the
+accuracy of the resulting sum, in the hope that all accumulated round-off
+error is deleted. However, this is unsafe since the order dependence remains
+and may even be amplified if the values of the sum lie around a boundary where
+values are rounded up or down when reducing the accuracy. Furthermore,
+problems arise if the summands have different signs and the result of
+summation consists of round-off error only.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html139"
+ HREF="node11.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html137"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html131"
+ HREF="node9.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html140"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+<B> Up:</B> <A NAME="tex2html138"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html132"
+ HREF="node9.html">Writing output detached to</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Parallel Jacobi preconditioning</TITLE>
+<META NAME="description" CONTENT="Parallel Jacobi preconditioning">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="previous" HREF="node10.html">
+<LINK REL="up" HREF="node8.html">
+<LINK REL="next" HREF="node12.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html147"
+ HREF="node12.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html145"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html141"
+ HREF="node10.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html148"
+ HREF="node12.html">Conclusions</A>
+<B> Up:</B> <A NAME="tex2html146"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html142"
+ HREF="node10.html">Assembling the matrix</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00043000000000000000">
+Parallel Jacobi preconditioning</A>
+</H2>
+
+<P>
+When preconditioning a matrix, for example in a Conjugate Gradients solver,
+one may choose the Jacobi scheme for preconditioning. The preconditioned
+vector <IMG
+ WIDTH="12" HEIGHT="15" ALIGN="BOTTOM" BORDER="0"
+ SRC="img2.gif"
+ ALT="$\tilde v$">
+is computed from the vector <I>v</I> using the following
+relationship:
+<BR><P></P>
+<DIV ALIGN="CENTER">
+<!-- MATH: \begin{displaymath}
+\tilde v_i = \frac 1{a_{ii}} v_i,
+\end{displaymath} -->
+
+
+<IMG
+ WIDTH="73" HEIGHT="41"
+ SRC="img3.gif"
+ ALT="\begin{displaymath}\tilde v_i = \frac 1{a_{ii}} v_i,
+\end{displaymath}">
+</DIV>
+<BR CLEAR="ALL">
+<P></P>
+where <I>a</I><SUB><I>ii</I></SUB> are the diagonal elements of the matrix which we are presently
+inverting. As is obvious, the result of preconditioning one element of <I>v</I> is
+entirely independent of all other elements, so this operation is trivially
+parallelizable. In practice, this is done by splitting the interval [0,<I>n</I>)into equal parts
+<!-- MATH: $[n_i,n_{i+1}), i=0,\dots,p-1$ -->
+<IMG
+ WIDTH="179" HEIGHT="31" ALIGN="MIDDLE" BORDER="0"
+ SRC="img4.gif"
+ ALT="$[n_i,n_{i+1}), i=0,\dots,p-1$">,
+where <I>n</I> is the size of the
+matrix, and <I>p</I> is the number of processors. Obviously,
+<!-- MATH: $n_0=0, n_p=n$ -->
+<I>n</I><SUB>0</SUB>=0, <I>n</I><SUB><I>p</I></SUB>=<I>n</I>, and
+
+<!-- MATH: $n_i<n_{i+1}$ -->
+<I>n</I><SUB><I>i</I></SUB><<I>n</I><SUB><I>i</I>+1</SUB>.
+
+<P>
+Just like for splitting a range of iterators using the function
+<TT>split_range</TT> used above, there is a function
+<PRE>
+ vector<pair<unsigned int, unsigned int> >
+ split_interval (const unsigned int &begin, const unsigned int &end,
+ const unsigned int n_intervals);
+</PRE>that splits the interval <TT>[begin,end)</TT> into <TT>n_intervals</TT> equal
+parts. This function will be used to assign each processor its share of
+elements <I>v</I><SUB><I>i</I></SUB>.
+
+<P>
+Furthermore, we will use some functionality provided by the
+<TT>MultithreadInfo</TT> class in <TT>deal.II</TT>. Upon start-up of the
+library, the static variable <TT>multithread_info.n_cpus</TT> is set to the
+number of processors in the computer the program is presently running on.
+<TT>multithread_info</TT> is a global variable of type
+<TT>MultithreadInfo</TT> available in all parts of the library. Furthermore,
+there is a variable <TT>multithread_info.n_default_threads</TT>, which by
+default is set to <TT>n_cpus</TT>, but which can be changed by the user; it
+denotes the default number of threads which the library shall use whenever
+multi-threading is implemented for some operation. We will use this variable to
+decide how many threads shall be used to precondition the vector.
+
+<P>
+The implementation of the preconditioning function then looks like this:
+<PRE>
+ // define an abbreviatory data type for an interval
+ typedef pair<unsigned int, unsigned int> Interval;
+
+ void Preconditioner::precondition_jacobi (const Matrix &m,
+ const Vector &v,
+ Vector &v_tilde) {
+ // define an abbreviation to the number
+ // of threads which we will use
+ const unsigned int n_threads = multithread_info.n_default_threads;
+ // first split the interval into equal pieces
+ vector<Interval> intervals = Threads::split_interval (0, m.rows(),
+ n_threads);
+
+ // then define a thread manager
+ ACE_Thread_Manager thread_manager;
+ // and finally start all the threads:
+ for (unsigned int i=0; i<n_threads; ++i)
+ Threads::spawn (thread_manager,
+ Threads::encapsulate (&Preconditioner::threaded_jacobi)
+ .collect_args (this, m, v, v_tilde, intervals[i]));
+
+ // wait for all the threads to finish
+ thread_manager.wait ();
+ };
+
+
+ void Preconditioner::threaded_jacobi (const Matrix &m,
+ const Vector &v,
+ Vector &v_tilde,
+ const Interval &interval) {
+ // apply the preconditioner in the given interval
+ for (unsigned int i=interval.first; i<interval.second; ++i)
+ v_tilde(i) = v(i) / m(i,i);
+ };
+</PRE>
+<P>
+It is noted, however, that more practical preconditioners are usually not
+easily parallelized. However, matrix-vector and vector-vector operations can
+often be reduced to independent parts and can then be implemented using
+multiple threads.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html147"
+ HREF="node12.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html145"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html141"
+ HREF="node10.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html148"
+ HREF="node12.html">Conclusions</A>
+<B> Up:</B> <A NAME="tex2html146"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html142"
+ HREF="node10.html">Assembling the matrix</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Conclusions</TITLE>
+<META NAME="description" CONTENT="Conclusions">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node13.html">
+<LINK REL="previous" HREF="node8.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node13.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html157"
+ HREF="node13.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html155"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html149"
+ HREF="node11.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html158"
+ HREF="node13.html">Bibliography</A>
+<B> Up:</B> <A NAME="tex2html156"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html150"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00050000000000000000">
+Conclusions</A>
+</H1>
+
+<P>
+We have shown how multi-threading is supported in <TT>deal.II</TT> and how it
+can be used in several examples occuring in common finite element programs. It
+was demonstrated that implementing a usable C++ interface poses several
+difficulties, both from the aspect of user friendliness as well as program
+correctness. In order to overcome these difficulties, first the more simple
+framework implemented in <TT>deal.II</TT> version 3.0 was discussed, followed
+by a rather complex scheme which will be the base of implementations in future
+versions.
+
+<P>
+The second framework features a more complicated hierarchy of classes as well
+as intricate use of templates and synchronization mechanisms, which however
+led to a design in which threads can be created in a user friendly, system
+independent, C++ like way suitable for common programs. The use of this
+framework is inherently safe and does not require special knowledge of the
+internals by the user, and is simple to use. By using it, the overhead
+required for programming multi-threaded applications is reduced to a minimum
+and the programmer can concentrate on the task of getting the semantics of
+multi-threaded programs right, in particular managing concurrent access to
+data and distributing work to different threads.
+
+<P>
+The framework has been used in several application programs and has shown that
+with only marginally increased programming effort, finite element programs can
+be made significantly faster on multi-processor machines.
+<BR>
+
+<P>
+
+<H4><A NAME="SECTION00050010000000000000">
+Acknowledgments.</A>
+</H4> The author would like to thank Thomas Richter
+for his work in parallelizing several parts of the <TT>deal.II</TT> library,
+and Ralf Hartmann for help in the preparation of this report.
+
+<P>
+
+<BR><HR>
+<!--Table of Child-Links-->
+<A NAME="CHILD_LINKS"> </A>
+<UL>
+<LI><A NAME="tex2html159"
+ HREF="node12.html#SECTION00050010000000000000">Acknowledgments.</A>
+</UL>
+<!--End of Table of Child-Links-->
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html157"
+ HREF="node13.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html155"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html149"
+ HREF="node11.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html158"
+ HREF="node13.html">Bibliography</A>
+<B> Up:</B> <A NAME="tex2html156"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html150"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Bibliography</TITLE>
+<META NAME="description" CONTENT="Bibliography">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node14.html">
+<LINK REL="previous" HREF="node12.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node14.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html168"
+ HREF="node14.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html166"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html160"
+ HREF="node12.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html169"
+ HREF="node14.html">About this document ...</A>
+<B> Up:</B> <A NAME="tex2html167"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html161"
+ HREF="node12.html">Conclusions</A>
+<BR><BR>
+<!--End of Navigation Panel-->
+<H2><A NAME="SECTIONREF">Bibliography</A>
+</H2>
+<DL COMPACT><DD><P></P><DT><A NAME="BK99tr"><STRONG>1</STRONG></A>
+<DD>
+Wolfgang Bangerth and Guido Kanschat.
+<BR>Concepts for object-oriented finite element software - the
+ <TT>deal.II</TT> library.
+<BR>Preprint 99-43, SFB 359, Universität Heidelberg, October 1999.
+
+<P></P><DT><A NAME="BK99m"><STRONG>2</STRONG></A>
+<DD>
+Wolfgang Bangerth and Guido Kanschat.
+<BR><EM><TT>deal.II</TT> Differential Equations Analysis Library,
+ Technical Reference</EM>.
+<BR>IWR Heidelberg, October 1999.
+<BR><TT>http://gaia.iwr.uni</TT>-<TT>heidelberg.de/~deal/</TT>.
+
+<P></P><DT><A NAME="Cus93"><STRONG>3</STRONG></A>
+<DD>
+H. Custer.
+<BR><EM>Inside Windows NT</EM>.
+<BR>Microsoft Press, Redmont, Washington, 1993.
+
+<P></P><DT><A NAME="ACE"><STRONG>4</STRONG></A>
+<DD>
+Douglas C. Schmidt et al.
+<BR>WWW homepage of the Adaptive Communications Environment
+ ACE, <TT>http://www.cs.wustl.edu/~schmidt/ACE.html</TT>.
+
+<P></P><DT><A NAME="SolarisThreads92"><STRONG>5</STRONG></A>
+<DD>
+J. Eykholt, S. Kleinman, S. Barton, R. Faulkner, A. Shivalingiah, M. Smith,
+ D. Stein, J. Voll, M. Weeks, and D. Williams.
+<BR>Beyon multiprocessing... Multithreading the SunOS kernel.
+<BR>In <EM>Proceedings of the Summer USENIX C++ Technical Conference,
+ San Antonio, Texas</EM>, June 1992.
+
+<P></P><DT><A NAME="PosixThreads96"><STRONG>6</STRONG></A>
+<DD>
+IEEE.
+<BR>Threads extensions for portable operating systems.
+<BR>Technical report, IEEE, 1995.
+
+<P></P><DT><A NAME="Sch94"><STRONG>7</STRONG></A>
+<DD>
+Douglas C. Schmidt.
+<BR>ACE: an object-oriented framework for developing distributed
+ applications.
+<BR>In <EM>Proceedings of the Sixth USENIX C++ Technical Conference,
+ Cambridge, Massachusetts</EM>. USENIX Association, April 1994.
+
+<P></P><DT><A NAME="SW95"><STRONG>8</STRONG></A>
+<DD>
+Douglas C. Schmidt and Nanbor Wang.
+<BR>An OO encapsulation of lightweight OS concurrency mechanisms
+ in the ACE toolkit.
+<BR>Technical Report WUCS-95-31, Washington University, St. Louis, 1995.
+</DL>
+
+<P>
+<BR><HR>
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>About this document ... </TITLE>
+<META NAME="description" CONTENT="About this document ... ">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="previous" HREF="node13.html">
+<LINK REL="up" HREF="index.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif_gr.gif">
+<A NAME="tex2html174"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html170"
+ HREF="node13.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Up:</B> <A NAME="tex2html175"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html171"
+ HREF="node13.html">Bibliography</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00070000000000000000">
+ About this document ... </A>
+</H1>
+ <P>
+This document was generated using the
+<A HREF="http://www-dsed.llnl.gov/files/programs/unix/latex2html/manual/"><STRONG>LaTeX</STRONG>2<tt>HTML</tt></A> translator Version 98.1p1 release (March 2nd, 1998)
+<P>
+Copyright © 1993, 1994, 1995, 1996, 1997,
+<A HREF="http://cbl.leeds.ac.uk/nikos/personal.html">Nikos Drakos</A>,
+Computer Based Learning Unit, University of Leeds.
+<P>
+The command line arguments were: <BR>
+ <STRONG>latex2html</STRONG> <tt>-split 5 -antialias multithreading.tex</tt>.
+<P>
+The translation was initiated by Wolfgang Bangerth on 2000-04-20
+<BR><HR>
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Threads</TITLE>
+<META NAME="description" CONTENT="Threads">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node3.html">
+<LINK REL="previous" HREF="node1.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node3.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html42"
+ HREF="node3.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html40"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html34"
+ HREF="node1.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html43"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Up:</B> <A NAME="tex2html41"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html35"
+ HREF="node1.html">Background</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00020000000000000000">
+Threads</A>
+</H1>
+
+<P>
+The basic entity for programming multi-processor machines are
+<I>threads</I>. They represent parts of the program which are executed in
+parallel. Threads can be considered as separate programs that work on the same
+main memory. On single-processor machines, they are simulated by letting each
+thread run for some time (usually a few milliseconds) before switching to the
+next thread. On multi-processor machines, threads can truly be executed in
+parallel. In order to let programs use more than one thread (which would be
+the regular sequential program), several aspects need to be covered:
+<UL>
+<LI>How do we assign operations to different threads? Of course, operations
+ which depend on each other must not be executed in reverse order. This can
+ be achieved by only letting independent operations run on different threads,
+ or by using synchronization methods. this is mostly a question of program
+ design and thus problem dependent, which is why both aspects will only be
+ briefly touched below.
+<LI>How does the operating system and the whole programming environment
+ support this?
+</UL>As mentioned, only the second aspect can be canonicalized, so we will treat it
+first. Some examples of actual parallelized applications are discussed in
+Section <A HREF="node8.html#sec:applications">4</A>.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html42"
+ HREF="node3.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html40"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html34"
+ HREF="node1.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html43"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Up:</B> <A NAME="tex2html41"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html35"
+ HREF="node1.html">Background</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Creating and managing threads</TITLE>
+<META NAME="description" CONTENT="Creating and managing threads">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node8.html">
+<LINK REL="previous" HREF="node2.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node4.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html52"
+ HREF="node4.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html50"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html44"
+ HREF="node2.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html53"
+ HREF="node4.html">Operating system dependence and</A>
+<B> Up:</B> <A NAME="tex2html51"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html45"
+ HREF="node2.html">Threads</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00030000000000000000">
+Creating and managing threads</A>
+</H1>
+
+<P>
+<BR><HR>
+<!--Table of Child-Links-->
+<A NAME="CHILD_LINKS"> </A>
+<UL>
+<LI><A NAME="tex2html54"
+ HREF="node4.html">Operating system dependence and ACE</A>
+<LI><A NAME="tex2html55"
+ HREF="node5.html"><TT>C</TT> interface to threads versus <TT>C++</TT></A>
+<LI><A NAME="tex2html56"
+ HREF="node6.html">First approach</A>
+<LI><A NAME="tex2html57"
+ HREF="node7.html">Second approach</A>
+<UL>
+<LI><A NAME="tex2html58"
+ HREF="node7.html#SECTION00034100000000000000">Finding the correct template parameters.</A>
+<LI><A NAME="tex2html59"
+ HREF="node7.html#SECTION00034200000000000000">Encapsulating the parameters.</A>
+<LI><A NAME="tex2html60"
+ HREF="node7.html#SECTION00034300000000000000">Using these objects.</A>
+<LI><A NAME="tex2html61"
+ HREF="node7.html#SECTION00034400000000000000">Virtual constructors.</A>
+<LI><A NAME="tex2html62"
+ HREF="node7.html#SECTION00034500000000000000">Spawning independent threads.</A>
+<LI><A NAME="tex2html63"
+ HREF="node7.html#SECTION00034600000000000000">Number of parameters. Non-member functions.</A>
+</UL></UL>
+<!--End of Table of Child-Links-->
+<BR><HR>
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Operating system dependence and ACE</TITLE>
+<META NAME="description" CONTENT="Operating system dependence and ACE">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node5.html">
+<LINK REL="previous" HREF="node3.html">
+<LINK REL="up" HREF="node3.html">
+<LINK REL="next" HREF="node5.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html72"
+ HREF="node5.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html70"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html64"
+ HREF="node3.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html73"
+ HREF="node5.html">C interface to threads</A>
+<B> Up:</B> <A NAME="tex2html71"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html65"
+ HREF="node3.html">Creating and managing threads</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00031000000000000000">
+Operating system dependence and ACE</A>
+</H2>
+
+<P>
+While all relevant operating systems now support multi-threaded programs, they
+all have different notions on what threads actually are on an operating system
+level, how they shall be managed and created. Even on Unix systems, which are
+usually well-standardized, there are at least three different and mutually
+incompatible interfaces to threads: POSIX threads [<A
+ HREF="node13.html#PosixThreads96">6</A>],
+Solaris threads [<A
+ HREF="node13.html#SolarisThreads92">5</A>], and Linux
+threads. Some operating systems support more than one interface, but there is
+no interface that is supported by all operating systems. Furthermore, other
+systems like Microsoft Windows have interfaces that are incompatible to all
+Unix systems [<A
+ HREF="node13.html#Cus93">3</A>].
+
+<P>
+Writing multi-threaded programs based on the operating system interfaces is
+therefore something inherently incompatible unless much effort is spent to
+port it to a new system. To avoid this, we chose to use the ACE (Adaptive
+Communication Environment, see [<A
+ HREF="node13.html#Sch94">7</A>,<A
+ HREF="node13.html#SW95">8</A>,<A
+ HREF="node13.html#ACE">4</A>]) library which encapsulates
+the operating system dependence and offers a uniform interface to the user.
+
+<P>
+We chose ACE over other libraries, since it runs on almost all relevant
+platforms, including most Unix systems and Microsoft Windows,
+and since it is to our knowledge the only library which is actively developed by a
+large group. Furthermore,
+it also is significantly larger than only thread management, offering
+interprocess communication and communication between different computers, as
+well as many other services. Contrary to most other libraries, it therefore
+offers both the ability to support a growing <TT>deal.II</TT> as well as the
+prospect to support independence also with respect to future platforms.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html72"
+ HREF="node5.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html70"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html64"
+ HREF="node3.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html73"
+ HREF="node5.html">C interface to threads</A>
+<B> Up:</B> <A NAME="tex2html71"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html65"
+ HREF="node3.html">Creating and managing threads</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>C interface to threads versus C++</TITLE>
+<META NAME="description" CONTENT="C interface to threads versus C++">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node6.html">
+<LINK REL="previous" HREF="node4.html">
+<LINK REL="up" HREF="node3.html">
+<LINK REL="next" HREF="node6.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html82"
+ HREF="node6.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html80"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html74"
+ HREF="node4.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html83"
+ HREF="node6.html">First approach</A>
+<B> Up:</B> <A NAME="tex2html81"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html75"
+ HREF="node4.html">Operating system dependence and</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00032000000000000000">
+<TT>C</TT> interface to threads versus <TT>C++</TT></A>
+</H2>
+
+<P>
+While ACE encapsulates almost all of the synchronization and interprocess
+interface into <TT>C++</TT> classes, it for some reason does not do so for
+thread creation. Rather it only offers the basic <TT>C</TT> interface:
+when creating a new thread, a function is called which has the following
+signature:<P>
+<DIV><A NAME="code:signature"><B>Code sample 1</B></A>
+<PRE>
+ void * f (void * arg);
+</PRE></DIV><P></P>
+Thus, only functions which take a single parameter of type <TT>void*</TT> and
+return a <TT>void*</TT> may be called. Further, these functions must be global
+or static member functions, as opposed to true member functions of
+classes. This is not in line with the <TT>C++</TT> philosophy and in fact does
+not fit well into <TT>deal.II</TT> as well: there is not a single function in
+the library that has this signature.
+
+<P>
+The task of multi-threading support in <TT>deal.II</TT> is therefore to
+encapsulate member functions, arbitrary types and numbers of parameters, and
+return types of functions into mechanisms built atop of ACE. This has been
+done twice for <TT>deal.II</TT>, and we will explain both approaches. At
+present, i.e. with version 3.0, only the first approach is distributed with
+<TT>deal.II</TT>, since the second is still experimental and due to the high
+complexity. The latter approach, however, has clear advantages over the first
+one, and it is planned to switch to it in the next major version of
+<TT>deal.II</TT>.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html82"
+ HREF="node6.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html80"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html74"
+ HREF="node4.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html83"
+ HREF="node6.html">First approach</A>
+<B> Up:</B> <A NAME="tex2html81"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html75"
+ HREF="node4.html">Operating system dependence and</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>First approach</TITLE>
+<META NAME="description" CONTENT="First approach">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node7.html">
+<LINK REL="previous" HREF="node5.html">
+<LINK REL="up" HREF="node3.html">
+<LINK REL="next" HREF="node7.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html92"
+ HREF="node7.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html90"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html84"
+ HREF="node5.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html93"
+ HREF="node7.html">Second approach</A>
+<B> Up:</B> <A NAME="tex2html91"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html85"
+ HREF="node5.html">C interface to threads</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00033000000000000000">
+First approach</A>
+</H2>
+
+<P>
+The first idea is the following: assume that we have a class
+<TT>TestClass</TT><P>
+<DIV><A NAME="code:testclass"><B>Code sample 2</B></A>
+<PRE>
+ class TestClass {
+ public:
+ void test_function (int i, double d);
+ };
+</PRE></DIV><P></P>
+and we would like to call
+<TT>test_object</TT>.<TT>test_function(1,3.1415926)</TT> on a newly created
+thread, where <TT>test_object</TT> is
+an object of type <TT>TestClass</TT>. We then need an object that encapsulates
+the address of the member function, a pointer to the object for which we want
+to call the function, and both parameters. This class would be suitable:<P>
+<DIV><A NAME="code:memfundata"><B>Code sample 3</B></A>
+<PRE>
+ struct MemFunData {
+ typedef void (TestClass::*MemFunPtr) (int, double);
+ MemFunPtr mem_fun_ptr;
+ TestClass *object;
+ int arg1;
+ double arg2;
+ };
+</PRE></DIV><P></P>
+
+<P>
+We further need a function that satisfies the signature required by the
+operating systems (or ACE, respectively), see Code Sample
+<A HREF="node5.html#code:signature">1</A>, and that can call the member function
+if we pass it an object of type <TT>MemFunData</TT>:<P>
+<DIV><A NAME="code:start_thread"><B>Code sample 4</B></A>
+<PRE>
+ void * start_thread (void *arg_ptr) {
+ // first reinterpret the void* as a
+ // pointer to the object which
+ // encapsulates the arguments
+ // and addresses:
+ MemFunData *mem_fun_data
+ = reinterpret_cast<MemFunData *>(arg_ptr);
+ // then call the member function:
+ (mem_fun_data->object)
+ ->*(mem_fun_data->mem_fun_ptr) (mem_fun_data->arg1,
+ mem_fun_data->arg2);
+ // since the function does not return
+ // a value, we do so ourselves:
+ return 0;
+ };
+</PRE></DIV><P></P>
+Such functions are called <I>trampoline functions</I> since they only serve
+as jump-off point for other functions.
+
+<P>
+We can then perform the desired call using the following sequence of commands:
+<PRE>
+ MemFunData mem_fun_data;
+ mem_fun_data.mem_fun_ptr = &TestClass::test_function;
+ mem_fun_data.object = &test_object;
+ mem_fun_data.arg1 = 1;
+ mem_fun_data.arg2 = 3.1415926;
+
+ ACE_Thread_Manager::spawn (&start_thread,
+ (void*)&mem_fun_data);
+</PRE><TT>ACE_Thread_Manager::spawn</TT> is the function from ACE that actually
+calls the operating system and tells it to create a new thread and call the
+function which it is given as first parameter (here: <TT>start_thread</TT>)
+with the parameter which is given as second parameter. <TT>start_thread</TT>,
+when called, will then get the address of the function which we wanted to call
+from its parameter, and call it with the values we wanted as arguments.
+
+<P>
+In practice, this would mean that we needed a structure like
+<TT>MemFunData</TT> and a function like <TT>start_thread</TT> for each class
+<TT>TestClass</TT> and all functions <TT>test_function</TT> with different
+signatures. This is clearly not feasible in practice and places an
+inappropriate burden on the programmer who wants to use multiple threads in
+his program. Fortunately, <TT>C++</TT> offers an elegant way for this problem,
+in the form of templates: we first define a data type which encapsulates
+address and arguments for all binary functions:<P>
+<DIV><A NAME="code:memfundata-template"><B>Code sample 5</B></A>
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ struct MemFunData {
+ typedef void (Class::*MemFunPtr) (Arg1, Arg2);
+ MemFunPtr mem_fun_ptr;
+ Class *object;
+ Arg1 arg1;
+ Arg2 arg2;
+ };
+</PRE></DIV><P></P>
+Next, we need a function that can process these arguments:<P>
+<DIV><A NAME="code:start_thread-template"><B>Code sample 6</B></A>
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ void * start_thread (void *arg_ptr) {
+ MemFunData<Class,Arg1,Arg2> *mem_fun_data
+ = reinterpret_cast<MemFunData<Class,Arg1,Arg2>*>(arg_ptr);
+ (mem_fun_data->object)
+ ->*(mem_fun_data->mem_fun_ptr) (mem_fun_data->arg1,
+ mem_fun_data->arg2);
+ return 0;
+ };
+</PRE></DIV><P></P>
+Then we can start the thread as follows:
+<PRE>
+ MemFunData<TestClass,int,double> mem_fun_data;
+ mem_fun_data.mem_fun_ptr = &TestClass::test_function;
+ mem_fun_data.object = &test_object;
+ mem_fun_data.arg1 = 1;
+ mem_fun_data.arg2 = 3.1415926;
+
+ ACE_Thread_Manager::spawn (&start_thread<TestClass,int,double>,
+ (void*)&mem_fun_data);
+</PRE>Here we first create an object which is suitable to encapsulate the parameters
+of a binary function that is a member function of the <TT>TestClass</TT> class
+and takes an integer and a double. Then we start the thread using the correct
+trampoline function. It is the user's responsibility to choose the correct
+trampoline function (i.e. to specify the correct template parameters) since
+the compiler only sees a <TT>void*</TT> and cannot do any type checking.
+
+<P>
+We can further simplify the process and remove the user responsibility by
+defining the following class and function:<P>
+<DIV><A NAME="code:threadmanager"><B>Code sample 7</B></A>
+<PRE>
+ class ThreadManager : public ACE_Thread_Manager {
+ public:
+ template <typename Class, typename Arg1, typename Arg2>
+ static void
+ spawn (MemFunData<Class,Arg1,Arg2> &MemFunData) {
+ ACE_Thread_Manager::spawn (&start_thread<Class,Arg1,Arg2>,
+ (void*)&MemFunData);
+ };
+ };
+</PRE></DIV><P></P>
+This way, we can call
+<PRE>
+ ThreadManager::spawn (mem_fun_data);
+</PRE>and the compiler will figure out which the right trampoline function is, since
+it knows the data type of <TT>mem_fun_data</TT> and therefore the values of
+the template parameters in the <TT>ThreadManager:: spawn</TT> function.
+
+<P>
+The way described above is basically the way which is used in <TT>deal.II</TT>
+version 3.0. Some care has to be paid to details, however. In particular,
+<TT>C++</TT> functions often pass references as arguments, which however are
+not assignable after initialization. Therefore, the <TT>MemFunData</TT> class
+needs to have a constructor, and arguments must be set through it. Assume, for
+example, <TT>TestClass</TT> had a second member function
+<PRE>
+ void f (int &i, double &d);
+</PRE>Then, we would have to use <TT>MemFunData<TestClass,int&,doubleSPMamp;></TT>,
+which in a form without templates would look like this:
+<PRE>
+ struct MemFunData {
+ typedef void (TestClass::*MemFunPtr) (int &, double &);
+ MemFunPtr mem_fun_ptr;
+ TestClass *object;
+ int &arg1;
+ double &arg2;
+ };
+</PRE>The compiler would require us to initialize the references to the two
+parameters at construction time of the <TT>MemFunData</TT> object, since
+it is not possible in <TT>C++</TT> to change to which object a reference
+points to after initialization. Adding a constructor to the
+<TT>MemFunData</TT> class would then enable us to write
+<PRE>
+ int i = 1;
+ double d = 3.1415926;
+ MemFunData<TestClass,int&,double&>
+ mem_fun_data (&test_object, i, d, &TestClass::f);
+</PRE>Non-reference arguments could then still be changed after construction. For
+historical reasons, the pointer to the member function is passed as last
+parameter here.
+
+<P>
+The last point is that this interface is only usable for functions with two
+parameters. Basically, the whole process has to be reiterated for any number
+of parameters which we want to support. In <TT>deal.II</TT>, we therefore have
+classes <TT>MemFunData0</TT> through <TT>MemFunData10</TT>, corresponding to
+member function that do not take parameters through functions that take ten
+parameters. Equivalently, we need the respective number of trampoline
+functions.
+
+<P>
+Additional thoughts need to be taken on virtual member functions and constant
+functions. While the first are handled by the compiler (member function
+pointers can also be to virtual functions, without explicitly stating so), the
+latter can be achieved by writing
+<TT>MemFunData<const TestClass,int,double></TT>, which would be the correct
+object if we had declared <TT>test_function</TT> constant.
+
+<P>
+Finally we note that it is often the case that one member function starts a
+new thread by calling another member function of the same object. Thus, the
+declaration most often used is the following:
+<PRE>
+ MemFunData<TestClass,int&,double&>
+ mem_fun_data (this, 1, 3.1415926, &TestClass::f);
+</PRE>Here, instead of an arbitrary <TT>test_object</TT>, the present object is
+used, which is represented by the <TT>this</TT> pointer.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html92"
+ HREF="node7.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html90"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html84"
+ HREF="node5.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html93"
+ HREF="node7.html">Second approach</A>
+<B> Up:</B> <A NAME="tex2html91"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html85"
+ HREF="node5.html">C interface to threads</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Second approach</TITLE>
+<META NAME="description" CONTENT="Second approach">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="previous" HREF="node6.html">
+<LINK REL="up" HREF="node3.html">
+<LINK REL="next" HREF="node8.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html100"
+ HREF="node8.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html98"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html94"
+ HREF="node6.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html101"
+ HREF="node8.html">Applications</A>
+<B> Up:</B> <A NAME="tex2html99"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html95"
+ HREF="node6.html">First approach</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+<!--Table of Child-Links-->
+<A NAME="CHILD_LINKS"><strong>Subsections</strong></A>
+<UL>
+<LI><A NAME="tex2html102"
+ HREF="node7.html#SECTION00034100000000000000">Finding the correct template parameters.</A>
+<LI><A NAME="tex2html103"
+ HREF="node7.html#SECTION00034200000000000000">Encapsulating the parameters.</A>
+<LI><A NAME="tex2html104"
+ HREF="node7.html#SECTION00034300000000000000">Using these objects.</A>
+<LI><A NAME="tex2html105"
+ HREF="node7.html#SECTION00034400000000000000">Virtual constructors.</A>
+<LI><A NAME="tex2html106"
+ HREF="node7.html#SECTION00034500000000000000">Spawning independent threads.</A>
+<LI><A NAME="tex2html107"
+ HREF="node7.html#SECTION00034600000000000000">Number of parameters. Non-member functions.</A>
+</UL>
+<!--End of Table of Child-Links-->
+<HR>
+
+<H2><A NAME="SECTION00034000000000000000"> </A>
+<A NAME="sec:secondapproach"> </A>
+<BR>
+Second approach
+</H2>
+
+<P>
+While the approach outlined above works satisfactorily, it has one serious
+drawback: the programmer has to provide the data types of the arguments of the
+member function himself. While this seems to be a simple task, in practice it
+is often not, as will be explained in the sequel.
+
+<P>
+To expose the problem, we take an example from one of our application programs
+where we would like to call the function
+<PRE>
+ template <int dim>
+ void DoFHandler<dim>::distribute_dofs (const FiniteElement<dim> &,
+ const unsigned int);
+</PRE>on a new thread. Correspondingly, we would need to use
+<PRE>
+ MemFunData2<DoFHandler<dim>, const FiniteElement<dim> &, unsigned int>
+ mem_fun_data (dof_handler, fe, 0,
+ &DoFHandler<dim>::distribute_dofs);
+</PRE>to encapsulate the parameters. However, if one forgets the <TT>const</TT>
+specifier on the second template parameter, one receives the following error
+message (using gcc 2.95.2):
+<PRE>
+ test.cc: In method `void InterstepData<2>::wake_up(unsigned int, Interst
+ epData<2>::PresentAction)':
+ test.cc:683: instantiated from here
+ test.cc:186: no matching function for call to `ThreadManager::Mem_Fun_Da
+ ta2<DoFHandler<2>,FiniteElement<2> &,unsigned int>::MemFunData2 (DoFHa
+ ndler<2> *, const FiniteElement<2> &, int, void (DoFHandler<2>::*)(const
+ FiniteElement<2> &, unsigned int))'
+ /home/atlas1/wolf/program/newdeal/deal.II/base/include/base/thread_manag
+ er.h:470: candidates are: ThreadManager::MemFunData2<DoFHandler<2>,Fin
+ iteElement<2> &,unsigned int>::MemFunData2(DoFHandler<2> *, FiniteElem
+ ent<2> &, unsigned int, void * (DoFHandler<2>::*)(FiniteElement<2> &, un
+ signed int))
+ /home/atlas1/wolf/program/newdeal/deal.II/base/include/base/thread_manag
+ er.h:480: ThreadManager::MemFunData2<DoFHandler<2>,Fin
+ iteElement<2> &,unsigned int>::MemFunData2(DoFHandler<2> *, FiniteElem
+ ent<2> &, unsigned int, void (DoFHandler<2>::*)(FiniteElement<2> &, unsi
+ gned int))
+ /home/atlas1/wolf/program/newdeal/deal.II/base/include/base/thread_manag
+ er.h:486: ThreadManager::MemFunData2<DoFHandler<2>,Fin
+ iteElement<2> &,unsigned int>::MemFunData2(const ThreadManager::Mem_Fu
+ n_Data2<DoFHandler<2>,FiniteElement<2> &,unsigned int> &)
+</PRE>
+<P>
+While the compiler is certainly right to complain, the message is not very
+helpful. Furthermore, since interfaces to functions sometimes change, for
+example by adding additional default parameters that do not show up in usual
+code, programs that used to compile do no more so with messages as shown
+above.
+
+<P>
+Due to the lengthy and complex error messages, even very experienced
+programmers usually need between five and ten minutes until they get an
+expression like this correct. In most cases, they don't get it right in the
+first attempt, so the time used for the right declaration dominates the whole
+setup of starting a new thread. To circumvent this bottleneck at least in most
+cases, we chose to implement a second strategy at encapsulating the parameters
+of member functions. This is done in several steps: first let the compiler
+find out about the right template parameters, then encapsulate the parameters,
+use the objects, and finally solve some technical problems with virtual
+constructors and locking of destruction. We will treat these steps
+sequentially in the following.
+
+<P>
+
+<H3><A NAME="SECTION00034100000000000000">
+Finding the correct template parameters.</A>
+</H3>
+<TT>C++</TT> offers the possibility of templatized functions that deduce their
+template arguments themselves. In fact, we have used them in the
+<TT>ThreadManager::spawn</TT> function in Code Sample <A HREF="node6.html#code:threadmanager">7</A>
+already. Here, this can be used as follows: assume we have a
+function encapsulation class
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ class MemFunData { ... };
+</PRE>as above, and a function
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ MemFunData<Class,Arg1,Arg2>
+ encapsulate (void (Class::*mem_fun_ptr)(Arg1, Arg2)) {
+ return MemFunData<Class,Arg1,Arg2> (mem_fun_ptr);
+ };
+</PRE>Then, if we call this function with the test class of Code Sample
+<A HREF="node6.html#code:testclass">2</A> like this:
+<PRE>
+ encapsulate (&TestClass::test_function);
+</PRE>it can unambiguously determine the template parameters to be
+<TT>Class=TestClass</TT>, <TT>Arg1=int</TT>, <TT>Arg2=double</TT>.
+
+<P>
+
+<H3><A NAME="SECTION00034200000000000000">
+Encapsulating the parameters.</A>
+</H3>
+We should not try to include the argument values for the new thread right
+away, for example by declaring <TT>encapsulate</TT> like this:
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ MemFunData<Class,Arg1,Arg2>
+ encapsulate (void (Class::*mem_fun_ptr)(Arg1, Arg2),
+ Arg1 arg1,
+ Arg2 arg2,
+ Class object) {
+ return MemFunData<Class,Arg1,Arg2> (mem_fun_ptr, object, arg1, arg2);
+ };
+</PRE>The reason is that for template functions, no parameter promotion is
+performed. Thus, if we called this function as in
+<PRE>
+ encapsulate (&TestClass::test_function,
+ 1, 3,
+ test_object);
+</PRE>then the compiler would refuse this since from the function pointer it must
+deduce that <TT>Arg2 = double</TT>, but from the parameter ``3'' it must assume
+that <TT>Arg2 = int</TT>. The resulting error message would be similarly lengthy
+as the one shown above.
+
+<P>
+One could instead write <TT>MemFunData</TT> like this:
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ class MemFunData {
+ public:
+ typedef void (Class::*MemFunPtr)(Arg1, Arg2);
+
+ MemFunData (MemFunPtr mem_fun_ptr_) {
+ mem_fun_ptr = mem_fun_ptr_;
+ };
+
+ void collect_args (Class *object_,
+ Arg1 arg1_,
+ Arg2 arg2_) {
+ object = object_;
+ arg1 = arg1_;
+ arg2 = arg2_;
+ };
+
+ MemFunPtr mem_fun_ptr;
+ Class *object;
+ Arg1 arg1;
+ Arg2 arg2;
+ };
+</PRE>One would then create an object of this type including the parameters to be
+passed as follows:
+<PRE>
+ encapsulate(&TestClass::test_function).collect_args(test_object, 1, 3);
+</PRE>Here, the first function call creates an object with the right template
+parameters and storing the member function pointer, and the second one,
+calling a member function, fills in the function arguments.
+
+<P>
+Unfortunately, this way does not work: if one or more of the parameter types
+is a reference, then the respective reference variable needs to be initialized
+by the constructor, not by <TT>collect_args</TT>. It needs to be known which
+object the reference references at construction time, since later on only the
+referenced object can be assigned, not the reference itself anymore.
+
+<P>
+Since we feel that we are close to a solution, we introduce one more
+indirection, which indeed will be the last one:<P>
+<DIV><B>Code sample 8</B>
+<PRE>
+ template <typename Class, typename Arg1, typename Arg2>
+ class MemFunData {
+ public:
+ typedef void (Class::*MemFunPtr)(Arg1, Arg2);
+
+ MemFunData (MemFunPtr mem_fun_ptr_,
+ Class *object_,
+ Arg1 arg1_,
+ Arg2 arg2_) :
+ mem_fun_ptr (mem_fun_ptr_),
+ object (object_),
+ arg1 (arg1_),
+ arg2 (arg2_) {};
+
+ MemFunPtr mem_fun_ptr;
+ Class *object;
+ Arg1 arg1;
+ Arg2 arg2;
+ };
+
+
+ template <typename Class, typename Arg1, typename Arg2>
+ struct ArgCollector {
+ typedef void (Class::*MemFunPtr)(Arg1, Arg2);
+
+ ArgCollector (MemFunPtr mem_fun_ptr_) {
+ mem_fun_ptr = mem_fun_ptr_;
+ };
+
+
+ MemFunData<Class,Arg1,Arg2>
+ collect_args (Class *object_,
+ Arg1 arg1_,
+ Arg2 arg2_) {
+ return MemFunData<Class,Arg1,Arg2> (mem_fun_ptr, object,
+ arg1, arg2);
+ };
+
+ MemFunPtr mem_fun_ptr;
+ };
+
+
+ template <typename Class, typename Arg1, typename Arg2>
+ ArgCollector<Class,Arg1,Arg2>
+ encapsulate (void (Class::*mem_fun_ptr)(Arg1, Arg2)) {
+ return ArgCollector<Class,Arg1,Arg2> (mem_fun_ptr);
+ };
+</PRE></DIV><P></P>
+
+<P>
+Now we can indeed write for the test class of Code Sample <A HREF="node6.html#code:testclass">2</A>:
+<PRE>
+ encapsulate(&TestClass::test_function).collect_args(test_object, 1, 3);
+</PRE>The first call creates an object of type <TT>ArgCollector<...></TT> with the
+right parameters and storing the member function pointer, while the second
+call, a call to a member function of that
+intermediate class, generates the final object we are interested in, including
+the member function pointer and all necessary parameters. Since
+<TT>collect_args</TT> already has its template parameters fixed from
+<TT>encapsulate</TT>, it can convert between data types.
+
+<P>
+
+<H3><A NAME="SECTION00034300000000000000">
+Using these objects.</A>
+</H3>
+Now we have an object of the correct type
+automatically generated, without the need to type in any template parameters
+by hand. What can we do with that? First, we can't assign it to a variable of
+that type, e.g. for use in several <TT>spawn</TT> commands:
+<PRE>
+ MemFunData mem_fun_data = encapsulate(...).collect_args(...);
+</PRE>Why? Since we would then have to write the data type of that variable by hand:
+the correct data type is not <TT>MemFunData</TT> as written above, but
+<TT>MemFunData<TestClass,int,double></TT>. Specifying all these template
+arguments was exactly what we wanted to avoid. However, we can do some such
+thing if the variable to which we assign the result is of a type which is a
+base class of <TT>MemFunData<...></TT>. Unfortunately, the data values that
+<TT>MemFunData<...></TT> encapsulates depend on the template parameters, so
+the respective variables in which we store the values can only be placed in
+the derived class and could not be copied when we assign the variable to a
+base class object, since that does not have these variables.
+
+<P>
+What can we do here? Assume we have the following class structure:<P>
+<DIV><B>Code sample 9</B>
+<PRE>
+ class FunDataBase {};
+
+ template <...> class MemFunData : public FunDataBase
+ { /* as above */ };
+
+ class FunEncapsulation {
+ public:
+ FunEncapsulation (FunDataBase *f)
+ : fun_data_base (f) {};
+ FunDataBase *fun_data_base;
+ };
+
+
+ template <typename Class, typename Arg1, typename Arg2>
+ FunEncapsulation
+ ArgCollector<Class,Arg1,Arg2>::collect_args (Class *object_,
+ Arg1 arg1_,
+ Arg2 arg2_) {
+ return new MemFunData<Class,Arg1,Arg2> (mem_fun_ptr, object,
+ arg1, arg2);
+ };
+</PRE></DIV><P></P>
+Note that in the return statement of the <TT>collect_args</TT> function,
+first a cast from <TT>MemFunData*</TT> to <TT>FunDataBase*</TT>, and then a
+constructor call to <TT>FunEncapsulation :: FunEncapsulation (FunDataBase*)</TT>
+was performed.
+
+<P>
+In the example above, the call to <TT>encapsulate(...).collect_args(...)</TT>
+generates an object of type <TT>FunEncapsulation</TT>, which in turn stores a
+pointer to an object of type <TT>FunDataBase</TT>, here to
+<TT>MemFunData<...></TT> with the correct template parameters. We can assign
+the result to a variable the type of which does not contain any template
+parameters any more, as desired:
+<PRE>
+ FunEncapsulation
+ fun_encapsulation = encapsulate (&TestClass::test_function)
+ .collect_args(test_object, 1, 3);
+</PRE>
+<P>
+But how can we start a thread with this object if we have lost the full
+information about the data types? This can be done as follows: add a variable
+to <TT>FunDataBase</TT> which contains the address of a function that knows
+what to do. This function is usually implemented in the derived classes, and
+its address is passed to the constructor:<P>
+<DIV><B>Code sample 10</B>
+<PRE>
+ class FunDataBase {
+ public:
+ typedef void * (*ThreadEntryPoint) (void *);
+
+ FunDataBase (ThreadEntryPoint t) :
+ thread_entry_point (t) {};
+
+ ThreadEntryPoint thread_entry_point;
+ };
+
+ template <...>
+ class MemFunData : public FunDataBase {
+ public:
+ // among other things, the constructor now does this:
+ MemFunData () :
+ FunDataBase (&start_thread) {};
+
+ static void * start_thread (void *args) {
+ // do the same as in Code Sample 4 above
+ }
+ };
+
+
+ void spawn (ACE_Thread_Manager &thread_manager,
+ FunEncapsulation &fun_encapsulation) {
+ thread_manager.spawn (*fun_encapsulation.fun_data_base
+ ->thread_entry_point,
+ &fun_data_base);
+ };
+</PRE></DIV><P></P>
+<TT>fun_encapsulation.fun_data_base->thread_entry_point</TT> is given
+by the derived class as that function that knows how to handle objects of the
+type which we are presently using. Thus, we can now write the whole sequence
+of function calls (assuming we have an object <TT>thread_manager</TT> of type
+<TT>ACE_Thread_Manager</TT>):
+<PRE>
+ FunEncapsulation
+ fun_encapsulation = encapsulate (&TestClass::test_function)
+ .collect_args(test_object, 1, 3);
+ spawn (thread_manager, fun_encapsulation);
+</PRE>This solves our problem in that no template parameters need to be specified by
+hand any more. The only source for lengthy compiler error messages is if the
+parameters to <TT>collect_args</TT> are in the wrong order or can not be
+casted to the parameters of the member function which we want to call. These
+problems, however, are much more unlikely in our experience, and are also much
+quicker sorted out.
+
+<P>
+
+<H3><A NAME="SECTION00034400000000000000">
+Virtual constructors.</A>
+</H3> While the basic techniques have been fully
+developed now, there are some aspects which we still have to take care of. The
+basic problem here is that the <TT>FunEncapsulation</TT> objects store a
+pointer to an object that was created using the <TT>new</TT> operator. To
+prevent a memory leak, we need to destroy this object at some time, preferably
+in the destructor of <TT>FunEncapsulation</TT>:
+<PRE>
+ FunEncapsulation::~FunEncapsulation () {
+ delete fun_data_base;
+ };
+</PRE>However, what happens if we have copied the object before? In particular, this
+is always the case using the functions above: <TT>collect_args</TT> generates
+a temporary object of type <TT>FunEncapsulation</TT>, but there could be other
+sources of copies as well. If we do not take special precautions, only the
+pointer to the object is copied around, and we end up with stale pointers
+pointing to invalid locations in memory once the first object has been
+destroyed. What we obviously need to do when copying objects of type
+<TT>FunEncapsulation</TT> is to not copy the pointer but to copy the object
+which it points to. Unfortunately, the following copy constructor is not
+possible:
+<PRE>
+ FunEncapsulation::FunEncapsulation (const FunEncapsulation &m) {
+ fun_data_base = new FunDataBase (*m.fun_data_base);
+ };
+</PRE>The reason, of course, is that we do not want to copy that part of the object
+belonging to the abstract base class. But we can emulate something like this
+in the following way (this programming idiom is called ``virtual
+constructors''): <P>
+<DIV><B>Code sample 11</B>
+<PRE>
+ class FunDataBase {
+ public:
+ // as above
+
+ virtual FunDataBase * clone () const = 0;
+ };
+
+ template <...>
+ class MemFunData : public FunDataBase {
+ public:
+ // as above
+
+ // copy constructor:
+ MemFunData (const MemFunData<...> &mem_fun_data) {...};
+
+ // clone the present object, i.e.
+ // create an exact copy:
+ virtual FunDataBase * clone () const {
+ return new MemFunData<...>(*this);
+ };
+ };
+
+
+ FunEncapsulation::FunEncapsulation (const FunEncapsulation &m) {
+ fun_data_base = m.fun_data_base->clone ();
+ };
+</PRE></DIV><P></P>
+Thus, whenever the <TT>FunEncapsulation</TT> object is copied, it creates a
+copy of the object it harbors (the <TT>MemFunData<...></TT> object), and
+therefore always owns its copy. When the destructor is called, it is free to
+delete its copy without affecting other objects (from which it may have been
+copied, or to which it was copied). Similar to the copy constructor, we have
+to modify the copy operator, as well.
+
+<P>
+
+<H3><A NAME="SECTION00034500000000000000">
+Spawning independent threads.</A>
+</H3>
+
+<P>
+Often, one wants to spawn a thread which will have its own existence until it
+finishes, but is in no way linked to the creating thread any more. An example
+would be the following, assuming a function
+<TT>TestClass::compress_file(const string file_name)</TT> exists and that
+there is an object <TT>thread_manager</TT> not local to this function:
+<PRE>
+
+ ...
+ string file_name;
+ ... // write some output to a file
+
+ // now create a thread which runs `gzip' on that output file to reduce
+ // disk space requirements. don't care about that thread any more
+ // after creation, i.e. don't wait for its return
+ FunEncapsulation
+ fun_encapsulation = encapsulate (&TestClass::compress_file)
+ .collect_args(test_object, file_name);
+ spawn (thread_manager, fun_encapsulation);
+
+ // quit the present function
+ return;
+</PRE>The problem here is that the object <TT>fun_encapsulation</TT> goes out
+of scope when we quit the present function, and therefore also deletes its
+pointer to the data which we need to start the new thread. If in this case the
+operating system was a bit lazy in creating the new thread, the function
+<TT>start_thread</TT> would at best find a pointer pointing to an object
+which is already deleted. Further, but this is obvious, if the function is
+taking references or pointers to other objects, it is to be made sure that
+these objects persist at least as long as the spawned thread runs.
+
+<P>
+What one would need to do here at least, is wait until the thread is started
+for sure, before deletion of the <TT>FunEncapsulation</TT> is allowed. To
+this end, we need to use a ``Mutex'', to allow for exclusive operations. A
+Mutex (short for <I>mut</I>ually <I>ex</I>clusive) is an object managed by
+the operating system and which can only be ``owned'' by one thread at a
+time. You can try to ``acquire'' a Mutex, and you can later ``release'' it. If
+you try to acquire it, but the Mutex is owned by another thread, then your
+thread is blocked until the present owner releases it. Mutices (plural of
+``Mutex'') are therefore most often used to guarantee that only one thread is
+presently accessing some object: a thread that wants to access that object
+acquires a Mutex related to that object and only releases it once the access
+if finished; if in the meantime another thread wants to access that object as
+well, it has to acquire the Mutex, but since the Mutex is presently owned
+already, the second thread is blocked until the first one has finished its
+access.
+
+<P>
+Alternatively, one can use Mutices to synchronize things. We will use it for
+the following purpose: the Mutex is acquired by the starting thread; when
+later the destructor of the <TT>FunEncapsulation</TT> class (running on the
+same thread) is called, it tries to acquire the lock again; it will thus only
+continue its operations once the Mutex has been released by someone, which we
+do on the spawned thread once we don't need the data of the
+<TT>FunEncapsulation</TT> object any more and destruction is safe.
+
+<P>
+All this can then be done in the following way:<P>
+<DIV><B>Code sample 12</B>
+<PRE>
+ class FunEncapsulation {
+ public:
+ ... // as before
+ ~FunEncapsulation ();
+ };
+
+
+ class FunDataBase {
+ public:
+ ... // as before
+ Mutex lock;
+ };
+
+ template <typename Class, typename Arg1, typename Arg2>
+ void * start_thread (void *arg_ptr) {
+ MemFunData<Class,Arg1,Arg2> *mem_fun_data
+ = reinterpret_cast<MemFunData *>(arg_ptr);
+
+ // copy the data arguments:
+ MemFunData<Class,Arg1,Arg2>::MemFunPtr
+ mem_fun_ptr = mem_fun_data->mem_fun_ptr;
+ Class * object = mem_fun_data->object;
+ Arg1 arg1 = mem_fun_data->arg1;
+ Arg2 arg2 = mem_fun_data->arg2;
+
+ // data is now copied, so the original object may be deleted:
+ mem_fun_data->lock.release ();
+
+ // now call the thread function:
+ object->*mem_fun_ptr (arg1, arg2);
+
+ return 0;
+ };
+
+
+ FunEncapsulation::~FunEncapsulation () {
+ // wait until the data is copied by the new thread and
+ // `release' is called by `start_thread':
+ fun_data_base->lock.acquire ();
+ // now delete the object which is no more needed
+ delete fun_data_base;
+ };
+
+
+ void spawn (ACE_Thread_Manager &thread_manager,
+ FunEncapsulation &fun_encapsulation) {
+ // lock the fun_encapsulation object
+ fun_encapsulation.fun_data_base->lock.acquire ();
+ thread_manager.spawn (*fun_encapsulation.fun_data_base
+ ->thread_entry_point,
+ &fun_data_base);
+ };
+</PRE></DIV><P></P>
+When we call <TT>spawn</TT>, we set a lock
+on the destruction of the <TT>FunEncapsulation</TT> object just before we
+start the new thread. This lock is only released when inside the new thread
+(i.e. inside the <TT>start_thread</TT> function) all arguments have been
+copied to a safe place. Now we have local copies and don't need the ones from the
+<TT>fun_encapsulation</TT> object any more, which we indicate by
+releasing the lock. Inside the destructor of that object, we wait until we can
+obtain the lock, which is only after it has been released by the newly started
+thread; after having waited till this moment, the destruction can go on
+safely, and we can exit the function from which the thread was started, if we
+like so.
+
+<P>
+The scheme just described also works if we start multiple threads using only
+one object of type <TT>FunEncapsulation</TT>:
+<PRE>
+ FunEncapsulation
+ fun_encapsulation = encapsulate (&TestClass::test_function)
+ .collect_args(test_object, arg_value);
+ spawn (thread_manager, fun_encapsulation);
+ spawn (thread_manager, fun_encapsulation);
+
+ // quit the present function
+ return;
+</PRE>Here, when starting the second thread the spawn function has to wait until the
+newly started first thread has released its lock on the object; however, this
+delay is small and should not pose a noticeable problem. Thus, no special
+treatment of this case is necessary, and we can in a simple way emulate the
+<TT>spawn_n</TT> function provided by most operating systems, which spawns
+several new threads at once:
+<PRE>
+ void spawn_n (ACE_Thread_Manager &thread_manager,
+ FunEncapsulation &fun_encapsulation,
+ const unsigned int n_threads) {
+ for (unsigned int i=0; i<n_threads; ++i)
+ spawn (thread_manager, fun_encapsulation);
+ };
+</PRE>A direct support of the <TT>spawn_n</TT> function of the operating system
+would be difficult, though, since each of the new threads would call
+<TT>lock.release()</TT>, even though the lock was only acquired once.
+
+<P>
+Since we have now made sure that objects are not deleted too early, even the
+following sequence is possible, which does not involve any named variables at
+all, only a temporary one, which immediately released after the call to
+<TT>spawn</TT>: <P>
+<DIV><B>Code sample 13</B>
+<PRE>
+ spawn (thread_manager,
+ encapsulate (&TestClass::test_function)
+ .collect_args(test_object, arg_value));
+</PRE></DIV><P></P>
+We most often use this very short idiom in the applications in Section
+<A HREF="node8.html#sec:applications">4</A> and in our own programs.
+
+<P>
+
+<H3><A NAME="SECTION00034600000000000000">
+Number of parameters. Non-member functions.</A>
+</H3>
+Above, we have explained how we can define classes for a binary member
+function. This approach is easily extended to member functions taking any
+number of parameters. We simply have to write classes <TT>MemFunData0</TT>,
+<TT>MemFunData1</TT>, and so on, which encapsulate member functions that take
+zero, one, etc parameters. Likewise, we have to have classes
+<TT>ArgCollectorN</TT> for each number of parameters, and functions
+<TT>encapsulate</TT> that return an object of type
+<TT>ArgCollectorN</TT>. Since functions can be overloaded on their argument
+types, we need not call the <TT>encapsulate</TT> functions differently.
+
+<P>
+All of which has been said above can also easily be adopted to global
+functions or static member functions. Instead of the classes
+<TT>MemFunDataN</TT> we can then use classes <TT>FunDataN</TT> that are also
+derived from <TT>FunDataBase</TT>. The respective <TT>ArgCollector</TT>
+classes then collect only the arguments, not the object on which we will
+operate. The class, <TT>FunEncapsulation</TT> is not affected by this, nor is
+<TT>FunDataBase</TT>.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html100"
+ HREF="node8.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html98"
+ HREF="node3.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html94"
+ HREF="node6.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html101"
+ HREF="node8.html">Applications</A>
+<B> Up:</B> <A NAME="tex2html99"
+ HREF="node3.html">Creating and managing threads</A>
+<B> Previous:</B> <A NAME="tex2html95"
+ HREF="node6.html">First approach</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Applications</TITLE>
+<META NAME="description" CONTENT="Applications">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node12.html">
+<LINK REL="previous" HREF="node3.html">
+<LINK REL="up" HREF="index.html">
+<LINK REL="next" HREF="node9.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html116"
+ HREF="node9.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html114"
+ HREF="index.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html108"
+ HREF="node7.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html117"
+ HREF="node9.html">Writing output detached to</A>
+<B> Up:</B> <A NAME="tex2html115"
+ HREF="index.html">No Title</A>
+<B> Previous:</B> <A NAME="tex2html109"
+ HREF="node7.html">Second approach</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H1><A NAME="SECTION00040000000000000000"> </A>
+<A NAME="sec:applications"> </A>
+<BR>
+Applications
+</H1>
+
+<P>
+In the next few subsections, we will show usual applications of multi-threading
+in the <TT>deal.II</TT> library. The programs already use the new scheme
+discussed in Section <A HREF="node7.html#sec:secondapproach">3.4</A>.
+
+<P>
+<BR><HR>
+<!--Table of Child-Links-->
+<A NAME="CHILD_LINKS"> </A>
+<UL>
+<LI><A NAME="tex2html118"
+ HREF="node9.html">Writing output detached to disk</A>
+<LI><A NAME="tex2html119"
+ HREF="node10.html">Assembling the matrix</A>
+<LI><A NAME="tex2html120"
+ HREF="node11.html">Parallel Jacobi preconditioning</A>
+</UL>
+<!--End of Table of Child-Links-->
+<BR><HR>
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>
--- /dev/null
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--Converted with LaTeX2HTML 98.1p1 release (March 2nd, 1998)
+originally by Nikos Drakos (nikos@cbl.leeds.ac.uk), CBLU, University of Leeds
+* revised and updated by: Marcus Hennecke, Ross Moore, Herb Swan
+* with significant contributions from:
+ Jens Lippmann, Marek Rouchal, Martin Wilck and others -->
+<HTML>
+<HEAD>
+<TITLE>Writing output detached to disk</TITLE>
+<META NAME="description" CONTENT="Writing output detached to disk">
+<META NAME="keywords" CONTENT="multithreading">
+<META NAME="resource-type" CONTENT="document">
+<META NAME="distribution" CONTENT="global">
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1">
+<LINK REL="STYLESHEET" HREF="multithreading.css">
+<LINK REL="next" HREF="node10.html">
+<LINK REL="previous" HREF="node8.html">
+<LINK REL="up" HREF="node8.html">
+<LINK REL="next" HREF="node10.html">
+</HEAD>
+<BODY >
+<!--Navigation Panel-->
+<A NAME="tex2html129"
+ HREF="node10.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html127"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html121"
+ HREF="node8.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html130"
+ HREF="node10.html">Assembling the matrix</A>
+<B> Up:</B> <A NAME="tex2html128"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html122"
+ HREF="node8.html">Applications</A>
+<BR>
+<BR>
+<!--End of Navigation Panel-->
+
+<H2><A NAME="SECTION00041000000000000000">
+Writing output detached to disk</A>
+</H2>
+
+<P>
+The output classes, i.e. basically the classes <TT>DataOut</TT> and
+<TT>DataOutStack</TT> and their base classes, follow a strictly hierarchical
+model of data flow. The two terminal classes know about such things as
+triangulations, degrees of freedom, or finite elements, but they translate
+this structured information into a rather simple intermediate format. This
+conversion is done in the <TT>build_patches</TT> functions of these
+classes. The actual output routines only convert this intermediate format into
+one of the supported graphics formats, which is then a relatively simple task.
+
+<P>
+This separation of processing of structured data and actual output of the
+intermediate format was chosen since the actual output routines became rather
+complex with growing scope of the whole library. For example, we had to update
+all output functions when vector-valued finite elements were supported, and we
+had to do so again when discontinuous elements were developed. This became an
+unmanageable burden with the growing number of output formats, and we decided
+that an intermediate format would be more appropriate, which is created by
+only one function, but can be written to output formats by a number of
+different functions.
+
+<P>
+In the present context, this has the following implications: once the
+intermediate data is created by the <TT>build_patches</TT> function, we need
+no more preserve the data from which it was made (i.e. the grid which it was
+computed on, or the vector holding the actual solution values) and we can go
+on with computing on the next finer grid, or the next time step, while the
+intermediate data is converted to a graphics format file detached from the
+main process. The only thing which we must make sure is that the program only
+terminates after all detached output threads are finished. This can be done in
+the following way:
+<PRE>
+ // somewhere define a thread manager that keeps track of all
+ // detached (`global') threads
+ ACE_Thread_Manager global_thread_manager;
+
+ // This is the class which does the computations:
+ class MainClass {
+ ...
+
+ // now two functions, the first is called from the main program
+ // for output, the second will manage detached output
+ void write_solution ();
+ void write_detached (DataOut<dim> *data_out);
+ };
+
+
+ void MainClass::write_solution () {
+ DataOut<dim> *data_out = new DataOut<dim>();
+
+ // attach DoFHandler, add data vectors, ...
+
+ data_out->build_patches ();
+
+ // now everything is in place, and we can write the data detached
+ // Note that we transfer ownership of `data_out' to the other thread
+ Threads::spawn (global_thread_manager,
+ Threads::encapsulate(&MainClass<dim>::write_detached)
+ .collect_args(this, data_out));
+ };
+
+
+ void MainClass::write_detached (DataOut<dim> *data_out) {
+ ofstream output_file ("abc");
+ data_out->write_gnuplot (output_file);
+
+ // now delete the object which we got from the starting thread
+ delete data_out;
+ };
+
+
+ int main () {
+ ... // do all the work
+
+ // now wait for all detached threads to finish
+ global_thread_manager.wait ();
+ };
+</PRE>
+<P>
+Note that the functions <TT>spawn</TT> and <TT>encapsulate</TT> are prefixed
+by <TT>Threads::</TT> since in the actual implementation in <TT>deal.II</TT>
+they are declared within a namespace of that name.
+
+<P>
+It should be noted that if you want to write output detached from the main
+thread, and from the main thread at the same time, you need a version of the
+C++ standard library delivered with your compiler that supports parallel
+output. For the GCC compiler, this can be obtained by configuring it with the
+flag <TT>-enable-threads</TT> at build time.
+
+<P>
+<HR>
+<!--Navigation Panel-->
+<A NAME="tex2html129"
+ HREF="node10.html">
+<IMG WIDTH="37" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="next"
+ SRC="../deal.II-paper/next_motif.gif"></A>
+<A NAME="tex2html127"
+ HREF="node8.html">
+<IMG WIDTH="26" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="up"
+ SRC="../deal.II-paper/up_motif.gif"></A>
+<A NAME="tex2html121"
+ HREF="node8.html">
+<IMG WIDTH="63" HEIGHT="24" ALIGN="BOTTOM" BORDER="0" ALT="previous"
+ SRC="../deal.II-paper/previous_motif.gif"></A>
+<BR>
+<B> Next:</B> <A NAME="tex2html130"
+ HREF="node10.html">Assembling the matrix</A>
+<B> Up:</B> <A NAME="tex2html128"
+ HREF="node8.html">Applications</A>
+<B> Previous:</B> <A NAME="tex2html122"
+ HREF="node8.html">Applications</A>
+<!--End of Navigation Panel-->
+<ADDRESS>
+<I>Wolfgang Bangerth</I>
+<BR><I>2000-04-20</I>
+</ADDRESS>
+</BODY>
+</HTML>