add BFGS minimizer

diff --git a/doc/news/changes/minor/20181226DenisDavydov b/doc/news/changes/minor/20181226DenisDavydov
new file mode 100644 (file)
index 0000000..e40be83
--- /dev/null
+++ b/doc/news/changes/minor/20181226DenisDavydov
@@ -0,0 +1,3 @@
+New: Add SolverBFGS to minimize a function using the limited memory BFGS approach.
+<br>
+(Denis Davydov, 2018/12/26)

diff --git a/include/deal.II/optimization/solver_bfgs.h b/include/deal.II/optimization/solver_bfgs.h
new file mode 100644 (file)
index 0000000..5f3ed06
--- /dev/null
+++ b/include/deal.II/optimization/solver_bfgs.h
@@ -0,0 +1,407 @@
+//-----------------------------------------------------------
+//
+//    Copyright (C) 2018 by the deal.II authors
+//
+//    This file is part of the deal.II library.
+//
+//    The deal.II library is free software; you can use it, redistribute
+//    it, and/or modify it under the terms of the GNU Lesser General
+//    Public License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//    The full text of the license can be found in the file LICENSE.md at
+//    the top level directory of deal.II.
+//
+//---------------------------------------------------------------
+
+#ifndef dealii_solver_bfgs_h
+#define dealii_solver_bfgs_h
+
+#include <deal.II/lac/solver.h>
+
+#include <deal.II/numerics/history.h>
+
+#include <deal.II/optimization/line_minimization.h>
+
+DEAL_II_NAMESPACE_OPEN
+
+/**
+ * Implement the limited memory BFGS minimization method.
+ *
+ * This class implements a method to minimize a given function for which only
+ * the values of the function and its derivatives, but not its second
+ * derivatives are available. The BFGS method is a variation of the Newton
+ * method for function minimization in which the Hessian matrix is only
+ * approximated. In particular, the Hessian is updated using the formula of
+ * Broyden, Fletcher, Goldfarb, and Shanno (BFGS):
+ * \f[
+ * H^{(k+1)} &= \left[
+ * I-\rho_{(k)} s^{(k)} \otimes y^{(k)}
+ * \right]
+ * H^{(k)}
+ * \left[
+ * I -\rho^{(k)} y^{(k)} \otimes s^{(k)}
+ * \right]
+ * +
+ * \rho^{(k)} s^{(k)} \otimes s^{(k)}  \\
+ * y^{(k)} &\dealcoloneq g^{(k+1)} - g^{(k)} \\
+ * s^{(k)} &\dealcoloneq x^{(k+1)} - x^{(k)} \\
+ * \rho^{(k)} &\dealcoloneq \frac{1}{y^{(k)} \cdot s^{(k)}}
+ * \f]
+ * for a symmetric positive definite $H$. Limited memory variant is
+ * implemented via the two-loop recursion.
+ *
+ * @author Denis Davydov, 2018
+ */
+template <typename VectorType>
+class SolverBFGS : public Solver<VectorType>
+{
+public:
+  /**
+   * Number type.
+   */
+  typedef typename VectorType::value_type Number;
+
+
+  /**
+   * Standardized data struct to pipe additional data to the solver.
+   */
+  struct AdditionalData
+  {
+    /**
+     * Constructor.
+     */
+    explicit AdditionalData(const unsigned int max_history_size = 5,
+                            const bool         debug_output     = false);
+
+    /**
+     * Maximum history size.
+     */
+    unsigned int max_history_size;
+
+    /**
+     * Print extra debug output to deallog.
+     */
+    bool debug_output;
+  };
+
+
+  /**
+   * Constructor.
+   */
+  explicit SolverBFGS(SolverControl &       residual_control,
+                      const AdditionalData &data = AdditionalData());
+
+  /**
+   * Solve the unconstrained minimization problem
+   * \f[
+   * \min_{\mathbf x} f(\mathbf x)
+   * \f]
+   * starting from initial state @p x.
+   *
+   * The function @p compute takes two arguments indicating the values of $x$
+   * and of the gradient $g=\nabla f(\mathbf x)=\frac{\partial f}{\partial
+   * \mathbf x}$. When called, it needs to update the gradient $g$ at the given
+   * location $x$ and return the value of the function being minimized, i.e.,
+   * $f(\mathbf x)$.
+   */
+  void
+  solve(
+    const std::function<Number(const VectorType &x, VectorType &g)> &compute,
+    VectorType &                                                     x);
+
+  /**
+   * Connect a slot to perform a custom line-search.
+   *
+   * Given the value of function @p f, the current value of unknown @p x,
+   * the gradient @p g and the search direction @p p,
+   * return the size $\alpha$ of the step $x \leftarrow x + \alpha p$,
+   * and update @p x, @p g and @p f accordingly.
+   */
+  boost::signals2::connection
+  connect_line_search_slot(
+    const std::function<
+      Number(Number &f, VectorType &x, VectorType &g, const VectorType &p)>
+      &slot);
+
+  /**
+   * Connect a slot to perform a custom preconditioning.
+   *
+   * The preconditioner is applied inside the two loop recursion to
+   * vector `g` using the history of position increments `s` and
+   * gradient increments `y`.
+   *
+   * One possibility is to use the oldest `s,y` pair:
+   * @code
+   *  const auto preconditioner = [](VectorType &                         g,
+   *                                 const FiniteSizeHistory<VectorType> &s,
+   *                                 const FiniteSizeHistory<VectorType> &y) {
+   *    if (s.size() > 0)
+   *      {
+   *        const unsigned int i  = s.size() - 1;
+   *        const auto         yy = y[i] * y[i];
+   *        const auto         sy = s[i] * y[i];
+   *        Assert(yy > 0 && sy > 0, ExcInternalError());
+   *        g *= sy / yy;
+   *      }
+   *  };
+   * @endcode
+   *
+   * No preconditioning is performed if the code using this class has not
+   * attached anything to the signal.
+   */
+  boost::signals2::connection
+  connect_preconditioner_slot(
+    const std::function<void(VectorType &                         g,
+                             const FiniteSizeHistory<VectorType> &s,
+                             const FiniteSizeHistory<VectorType> &y)> &slot);
+
+
+protected:
+  /**
+   * Additional data to the solver.
+   */
+  const AdditionalData additional_data;
+
+  /**
+   * Signal used to perform line search.
+   */
+  boost::signals2::signal<
+    Number(Number &f, VectorType &x, VectorType &g, const VectorType &p)>
+    line_search_signal;
+
+  /**
+   * Signal used to perform preconditioning.
+   */
+  boost::signals2::signal<void(VectorType &                         g,
+                               const FiniteSizeHistory<VectorType> &s,
+                               const FiniteSizeHistory<VectorType> &y)>
+    preconditioner_signal;
+};
+
+
+// -------------------  inline and template functions ----------------
+#ifndef DOXYGEN
+
+template <typename VectorType>
+SolverBFGS<VectorType>::AdditionalData::AdditionalData(
+  const unsigned int max_history_size_,
+  const bool         debug_output_)
+  : max_history_size(max_history_size_)
+  , debug_output(debug_output_)
+{}
+
+
+
+template <typename VectorType>
+SolverBFGS<VectorType>::SolverBFGS(SolverControl &       solver_control,
+                                   const AdditionalData &data)
+  : Solver<VectorType>(solver_control)
+  , additional_data(data)
+{}
+
+
+
+template <class VectorType>
+boost::signals2::connection
+SolverBFGS<VectorType>::connect_line_search_slot(
+  const std::function<
+    Number(Number &f, VectorType &x, VectorType &g, const VectorType &p)> &slot)
+{
+  Assert(line_search_signal.empty(),
+         ExcMessage("One should not attach more than one line search signal."));
+  return line_search_signal.connect(slot);
+}
+
+
+
+template <class VectorType>
+boost::signals2::connection
+SolverBFGS<VectorType>::connect_preconditioner_slot(
+  const std::function<void(VectorType &                         g,
+                           const FiniteSizeHistory<VectorType> &s,
+                           const FiniteSizeHistory<VectorType> &y)> &slot)
+{
+  Assert(preconditioner_signal.empty(),
+         ExcMessage(
+           "One should not attach more than one preconditioner signal."));
+  return preconditioner_signal.connect(slot);
+}
+
+
+
+template <typename VectorType>
+void
+SolverBFGS<VectorType>::solve(
+  const std::function<typename VectorType::value_type(const VectorType &x,
+                                                      VectorType &f)> &compute,
+  VectorType &                                                         x)
+{
+  // Also see scipy Fortran implementation
+  // https://github.com/scipy/scipy/blob/master/scipy/optimize/lbfgsb_src/lbfgsb.f
+  // and Octave-optim implementation:
+  // https://sourceforge.net/p/octave/optim/ci/default/tree/src/__bfgsmin.cc
+  LogStream::Prefix prefix("BFGS");
+
+  // default line search:
+  bool   first_step = true;
+  Number f_prev     = 0.;
+  // provide default line search if no signal was attached
+  VectorType x0;
+  if (line_search_signal.empty())
+    {
+      x0.reinit(x);
+      const auto default_line_min =
+        [&](Number &f, VectorType &x, VectorType &g, const VectorType &p) {
+          const Number f0 = f;
+          const Number g0 = g * p;
+          Assert(g0 < 0,
+                 ExcMessage(
+                   "Function does not decrease along the current direction"));
+
+          // save current solution value (to be used in line_search):
+          x0 = x;
+
+          // see scipy implementation
+          // https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.line_search.html#scipy.optimize.line_search
+          // and Eq. 2.6.8 in Fletcher 2013, Practical methods of optimization
+          Number df = f_prev - f;
+          Assert(first_step || df >= 0.,
+                 ExcMessage("Function value is not decreasing"));
+          df = std::max(df, 100. * std::numeric_limits<Number>::epsilon());
+          // guess a reasonable first step:
+          const Number a1 =
+            (first_step ? 1. : std::min(1., -1.01 * 2. * df / g0));
+          Assert(a1 > 0., ExcInternalError());
+          f_prev = f;
+
+          // 1D line-search function
+          const auto line_func =
+            [&](const Number &x_line) -> std::pair<Number, Number> {
+            x = x0;
+            x.add(x_line, p);
+            f                   = compute(x, g);
+            const Number g_line = g * p;
+            return std::make_pair(f, g_line);
+          };
+
+          // loose line search:
+          const auto res = LineMinimization::line_search<Number>(
+            line_func,
+            f0,
+            g0,
+            LineMinimization::poly_fit<Number>,
+            a1,
+            0.9,
+            0.001);
+
+          if (first_step)
+            first_step = false;
+
+          return res.first;
+        };
+      this->connect_line_search_slot(default_line_min);
+    }
+
+  // FIXME: Octave has convergence in terms of:
+  // function change tolerance, default 1e-12
+  // parameter change tolerance, default 1e-6
+  // gradient tolerance, default 1e-5
+  // SolverBase and/or SolverControl need extension
+
+  VectorType g(x), p(x), y_k(x), s_k(x);
+
+  std::vector<Number> c1;
+  c1.reserve(additional_data.max_history_size);
+
+  // limited history
+  FiniteSizeHistory<VectorType> y(additional_data.max_history_size);
+  FiniteSizeHistory<VectorType> s(additional_data.max_history_size);
+  FiniteSizeHistory<Number>     rho(additional_data.max_history_size);
+
+  unsigned int m = 0;
+  Number       f;
+
+  SolverControl::State conv = SolverControl::iterate;
+  unsigned int         k    = 0;
+
+  f = compute(x, g);
+
+  conv = this->iteration_status(k, g.l2_norm(), x);
+  if (conv != SolverControl::iterate)
+    return;
+
+  while (conv == SolverControl::iterate)
+    {
+      if (additional_data.debug_output)
+        deallog << "Iteration " << k << " history " << m << std::endl
+                << "f=" << f << std::endl;
+
+      // 1. Two loop recursion to calculate p = - H*g
+      c1.resize(m);
+      p = g;
+      // first loop:
+      for (unsigned int i = 0; i < m; ++i)
+        {
+          c1[i] = rho[i] * (s[i] * p);
+          p.add(-c1[i], y[i]);
+        }
+      // H0
+      if (!preconditioner_signal.empty())
+        preconditioner_signal(p, s, y);
+
+      // second loop:
+      for (int i = m - 1; i >= 0; --i)
+        {
+          Assert(i >= 0, ExcInternalError());
+          const Number c2 = rho[i] * (y[i] * p);
+          p.add(c1[i] - c2, s[i]);
+        }
+      p *= -1.;
+
+      // 2. Line search
+      s_k                = x;
+      y_k                = g;
+      const Number alpha = line_search_signal(f, x, g, p)
+                             .get(); // <-- signals return boost::optional
+      s_k.sadd(-1, 1, x);
+      y_k.sadd(-1, 1, g);
+
+      if (additional_data.debug_output)
+        deallog << "Line search a=" << alpha << " f=" << f << std::endl;
+
+      // 3. Check convergence
+      k++;
+      const Number g_l2 = g.l2_norm();
+      conv              = this->iteration_status(k, g_l2, x);
+      if (conv != SolverControl::iterate)
+        break;
+
+      // 4. Store s, y, rho
+      const Number curvature = s_k * y_k;
+      if (additional_data.debug_output)
+        deallog << "Curvature " << curvature << std::endl;
+
+      if (curvature > 0. && additional_data.max_history_size > 0)
+        {
+          s.add(s_k);
+          y.add(y_k);
+          rho.add(1. / curvature);
+          m = s.size();
+
+          Assert(y.size() == m, ExcInternalError());
+          Assert(rho.size() == m, ExcInternalError());
+        }
+
+      Assert(m <= additional_data.max_history_size, ExcInternalError());
+    }
+
+  // In the case of failure: throw exception.
+  AssertThrow(conv == SolverControl::success,
+              SolverControl::NoConvergence(k, g.l2_norm()));
+}
+
+#endif
+
+DEAL_II_NAMESPACE_CLOSE
+
+#endif

diff --git a/tests/optimization/bfgs_03.cc b/tests/optimization/bfgs_03.cc
new file mode 100644 (file)
index 0000000..9853dce
--- /dev/null
+++ b/tests/optimization/bfgs_03.cc
@@ -0,0 +1,160 @@
+//-----------------------------------------------------------
+//
+//    Copyright (C) 2018 by the deal.II authors
+//
+//    This file is part of the deal.II library.
+//
+//    The deal.II library is free software; you can use it, redistribute
+//    it, and/or modify it under the terms of the GNU Lesser General
+//    Public License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//    The full text of the license can be found in the file LICENSE.md at
+//    the top level directory of deal.II.
+//
+//---------------------------------------------------------------
+
+// test limited memory BFGS with quadratic function
+// f(x) = 0.5 x*Lx - x*f
+// f'(x) = Lx - f
+// where L is 1d FD Laplacian
+//
+// We compare results to the companion Octave file which uses BFGS
+// from optim package. The output of this test is made similar to the
+// Octave output.
+
+
+#include <deal.II/base/logstream.h>
+
+#include <deal.II/lac/full_matrix.h>
+#include <deal.II/lac/vector.h>
+
+#include <deal.II/optimization/solver_bfgs.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "../tests.h"
+
+template <typename number>
+void
+test()
+{
+  typedef Vector<number> VectorType;
+
+  // size of the problem
+  const unsigned int N = 10;
+
+  // parameters:
+  const unsigned int itmax = 100;
+  const double       tol   = 1e-8;
+
+  // 1D Laplace with zero Dirichlet BC on both sides
+  FullMatrix<number> L(N);
+  L = 0.;
+
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      if (i > 0)
+        L(i, i - 1) = -1.;
+      L(i, i) = 2.;
+      if (i < N - 1)
+        L(i, i + 1) = -1.;
+    }
+
+  // L.print_formatted(deallog.get_file_stream(), 6, false, 10);
+
+  // RHS
+  VectorType b(N);
+  for (unsigned int i = 0; i < N; ++i)
+    b(i) = 1.;
+
+  // solution
+  VectorType x(N);
+  x = 1.;
+
+  // safety measure to not modify L or b within Lambda.
+  const FullMatrix<number> &L_const = L;
+  const VectorType &        b_const = b;
+  const auto                func    = [&](const VectorType &x, VectorType &g) {
+    L_const.vmult(g, x);
+    number res = 0.5 * (g * x) - x * b_const;
+    g.add(-1, b);
+    return res;
+  };
+
+  // exact line minimization for quadratic function
+  /*
+  f(x) := a*x**2 + b*x + c;
+  g(x) := ''(diff(f(x),x));
+  sol : solve([f(0)=f0, f(1)=f1, g(0)=g0],[a,b,c]);
+  sol2 : solve(diff(f(u),u)=0,u);
+  subst(sol,sol2);
+
+  u=-g0/(2*(-g0+f1-f0))
+
+  */
+
+  bool first_step = true;
+
+  int        iteration = 0;
+  VectorType dx(x);
+
+  const auto line_min =
+    [&](number &f, VectorType &x, VectorType &g, const VectorType &p) {
+      deallog << "-------------------" << std::endl
+              << "Line search " << iteration++ << ":" << std::endl;
+
+      const number g_norm_sqr = g.norm_sqr();
+
+      deallog << "Gradient:" << std::endl;
+      g.print(deallog.get_file_stream(), 5, false);
+
+      // directional derivative
+      const number df = g * p;
+      Assert(df < 0, ExcInternalError());
+      // do the full step
+      x.add(1., p);
+      // save old value
+      const number f_old = f;
+      // calculate new value
+      f = func(x, g);
+      // get the step size
+      const number denom = -df + f - f_old;
+      Assert(denom != 0., ExcDivideByZero());
+      Assert(denom > 0, ExcInternalError());
+      const number step = -df * 0.5 / denom;
+      // do the step
+      x.add(step - 1., p);
+      f = func(x, g);
+
+      first_step = false;
+      deallog << "Function value: " << f_old << " stepsize: " << step
+              << " new value: " << f << std::endl;
+      deallog << "Change:" << std::endl;
+      dx = p;
+      dx *= step;
+      dx.print(deallog.get_file_stream(), 5, false);
+
+      // finally return the step size
+      return step;
+    };
+
+  SolverControl solver_control(itmax, tol, true);
+  typename SolverBFGS<VectorType>::AdditionalData data(10, false);
+  SolverBFGS<VectorType>                          solver(solver_control, data);
+  solver.connect_line_search_slot(line_min);
+  solver.solve(func, x);
+
+  deallog << "Limited memory BFGS solution:" << std::endl;
+  x.print(deallog);
+}
+
+int
+main()
+{
+  std::ofstream logfile("output");
+  deallog << std::setprecision(5);
+  deallog.attach(logfile);
+
+  test<double>();
+}

diff --git a/tests/optimization/bfgs_03.output b/tests/optimization/bfgs_03.output
new file mode 100644 (file)
index 0000000..202cf2f
--- /dev/null
+++ b/tests/optimization/bfgs_03.output
@@ -0,0 +1,46 @@
+
+DEAL:BFGS::Check 0     2.8284
+DEAL:BFGS::Starting value 2.8284
+DEAL:BFGS::-------------------
+DEAL:BFGS::Line search 0:
+DEAL:BFGS::Gradient:
+0.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 0.00000 
+DEAL:BFGS::Function value: -9.0000 stepsize: 4.0000 new value: -25.000
+DEAL:BFGS::Change:
+0.00000 4.00000 4.00000 4.00000 4.00000 4.00000 4.00000 4.00000 4.00000 0.00000 
+DEAL:BFGS::Check 1     7.4833
+DEAL:BFGS::-------------------
+DEAL:BFGS::Line search 1:
+DEAL:BFGS::Gradient:
+-4.00000 3.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 -1.00000 3.00000 -4.00000 
+DEAL:BFGS::Function value: -25.000 stepsize: 0.87500 new value: -49.500
+DEAL:BFGS::Change:
+3.50000 3.50000 7.00000 7.00000 7.00000 7.00000 7.00000 7.00000 3.50000 3.50000 
+DEAL:BFGS::Check 2     4.1833
+DEAL:BFGS::-------------------
+DEAL:BFGS::Line search 2:
+DEAL:BFGS::Gradient:
+-0.50000 -0.50000 2.50000 -1.00000 -1.00000 -1.00000 -1.00000 2.50000 -0.50000 -0.50000 
+DEAL:BFGS::Function value: -49.500 stepsize: 0.47619 new value: -53.667
+DEAL:BFGS::Change:
+0.83333 0.83333 0.00000 1.66667 1.66667 1.66667 1.66667 0.00000 0.83333 0.83333 
+DEAL:BFGS::Check 3     1.8257
+DEAL:BFGS::-------------------
+DEAL:BFGS::Line search 3:
+DEAL:BFGS::Gradient:
+0.33333 0.33333 0.00000 0.66667 -1.00000 -1.00000 0.66667 0.00000 0.33333 0.33333 
+DEAL:BFGS::Function value: -53.667 stepsize: 0.60000 new value: -54.667
+DEAL:BFGS::Change:
+0.00000 0.00000 0.00000 0.00000 1.00000 1.00000 0.00000 0.00000 0.00000 0.00000 
+DEAL:BFGS::Check 4     0.81650
+DEAL:BFGS::-------------------
+DEAL:BFGS::Line search 4:
+DEAL:BFGS::Gradient:
+0.33333 0.33333 0.00000 -0.33333 0.00000 0.00000 -0.33333 0.00000 0.33333 0.33333 
+DEAL:BFGS::Function value: -54.667 stepsize: 1.0000 new value: -55.000
+DEAL:BFGS::Change:
+-0.33333 -0.33333 0.00000 0.33333 0.33333 0.33333 0.33333 0.00000 -0.33333 -0.33333 
+DEAL:BFGS::Check 5     3.5572e-14
+DEAL:BFGS::Convergence step 5 value 3.5572e-14
+DEAL::Limited memory BFGS solution:
+DEAL::5.0000 9.0000 12.000 14.000 15.000 15.000 14.000 12.000 9.0000 5.0000 

diff --git a/tests/optimization/bfgs_04.cc b/tests/optimization/bfgs_04.cc
new file mode 100644 (file)
index 0000000..2052447
--- /dev/null
+++ b/tests/optimization/bfgs_04.cc
@@ -0,0 +1,163 @@
+//-----------------------------------------------------------
+//
+//    Copyright (C) 2018 by the deal.II authors
+//
+//    This file is part of the deal.II library.
+//
+//    The deal.II library is free software; you can use it, redistribute
+//    it, and/or modify it under the terms of the GNU Lesser General
+//    Public License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//    The full text of the license can be found in the file LICENSE.md at
+//    the top level directory of deal.II.
+//
+//---------------------------------------------------------------
+
+// test limited memory BFGS with quadratic function
+// f(x) = 0.5 x*Lx - x*f
+// f'(x) = Lx - f
+// where L is 1d FD Laplacian
+// Same problem as in bfgs_03, but with
+// larger size and lower history.
+
+
+#include <deal.II/base/logstream.h>
+
+#include <deal.II/lac/full_matrix.h>
+#include <deal.II/lac/vector.h>
+
+#include <deal.II/optimization/solver_bfgs.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "../tests.h"
+
+template <typename number>
+void
+test()
+{
+  auto &out = deallog.get_file_stream();
+  out << std::setprecision(5) << std::fixed << std::right;
+
+  typedef Vector<number> VectorType;
+
+  // size of the problem
+  const unsigned int N = 20;
+
+  // parameters:
+  const unsigned int itmax = 100;
+  const double       tol   = 1e-8;
+  const unsigned int m_max = 4;
+
+  // 1D Laplace with zero Dirichlet BC on both sides
+  FullMatrix<number> L(N);
+  L = 0.;
+
+  for (unsigned int i = 0; i < N; ++i)
+    {
+      if (i > 0)
+        L(i, i - 1) = -1.;
+      L(i, i) = 2.;
+      if (i < N - 1)
+        L(i, i + 1) = -1.;
+    }
+
+  // L.print_formatted(deallog.get_file_stream(), 6, false, 10);
+
+  // RHS
+  VectorType b(N);
+  for (unsigned int i = 0; i < N; ++i)
+    b(i) = 1.;
+
+  // solution
+  VectorType x(N);
+  x = 1.;
+
+  // safety measure to not modify L or b within Lambda.
+  const FullMatrix<number> &L_const = L;
+  const VectorType &        b_const = b;
+  const auto                func    = [&](const VectorType &x, VectorType &g) {
+    L_const.vmult(g, x);
+    number res = 0.5 * (g * x) - x * b_const;
+    g.add(-1, b);
+    return res;
+  };
+
+  // exact line minimization for quadratic function
+  /*
+  f(x) := a*x**2 + b*x + c;
+  g(x) := ''(diff(f(x),x));
+  sol : solve([f(0)=f0, f(1)=f1, g(0)=g0],[a,b,c]);
+  sol2 : solve(diff(f(u),u)=0,u);
+  subst(sol,sol2);
+
+  u=-g0/(2*(-g0+f1-f0))
+
+  */
+
+  int        iteration = 0;
+  VectorType dx(x), old_x(x), old_g(x);
+
+  const auto line_min =
+    [&](number &f, VectorType &x, VectorType &g, const VectorType &p) {
+      out << "------------------------------------------------" << std::endl
+          << "Line search " << iteration++ << std::endl
+          << std::endl;
+
+      const number g_norm_sqr = g.norm_sqr();
+
+      old_x = x;
+      old_g = g;
+
+      // directional derivative
+      const number df = g * p;
+      Assert(df < 0, ExcInternalError());
+      // do the full step
+      x.add(1., p);
+      // save old value
+      const number f_old = f;
+      // calculate new value
+      f = func(x, g);
+      // get the step size
+      const number denom = -df + f - f_old;
+      Assert(denom != 0., ExcDivideByZero());
+      Assert(denom > 0, ExcInternalError());
+      const number step = -df * 0.5 / denom;
+      // do the step
+      x.add(step - 1., p);
+      f = func(x, g);
+
+      out << "function value: " << f_old << "  stepsize: " << step << std::endl
+          << std::endl;
+      dx = p;
+      dx *= step;
+
+      const std::string s = "        ";
+      for (unsigned int i = 0; i < N; ++i)
+        out << s << std::setw(9) << old_x(i) << s << std::setw(9) << old_g(i)
+            << s << std::setw(9) << dx(i) << std::endl;
+
+      // finally return the step size
+      return step;
+    };
+
+  SolverControl solver_control(itmax, tol, true);
+  typename SolverBFGS<VectorType>::AdditionalData data(m_max, false);
+  SolverBFGS<VectorType>                          solver(solver_control, data);
+  solver.connect_line_search_slot(line_min);
+  solver.solve(func, x);
+
+  deallog << "Limited memory BFGS solution:" << std::endl;
+  x.print(deallog);
+}
+
+int
+main()
+{
+  std::ofstream logfile("output");
+  deallog << std::setprecision(5);
+  deallog.attach(logfile);
+
+  test<double>();
+}

diff --git a/tests/optimization/bfgs_04.m b/tests/optimization/bfgs_04.m
new file mode 100755 (executable)
index 0000000..0a3b115
--- /dev/null
+++ b/tests/optimization/bfgs_04.m
@@ -0,0 +1,91 @@
+% //-----------------------------------------------------------
+% //
+% //    Copyright (C) 2018 by the deal.II authors
+% //
+% //    This file is part of the deal.II library.
+% //
+% //    The deal.II library is free software; you can use it, redistribute
+% //    it, and/or modify it under the terms of the GNU Lesser General
+% //    Public License as published by the Free Software Foundation; either
+% //    version 2.1 of the License, or (at your option) any later version.
+% //    The full text of the license can be found in the file LICENSE.md at
+% //    the top level directory of deal.II.
+% //
+% //---------------------------------------------------------------
+
+% a companion to bfgs_04.cc which minimizes the same function using Octave.
+
+% bfgs_04.cc matches the Octave. Strangely enough "stepsize" in Octave differs starting from
+% iteration 4, however the value, gradient and increment in value (three columns) match perfectly.
+
+% This file uses bfgsmin() from optim package: https://octave.sourceforge.io/optim/
+% https://octave.sourceforge.io/optim/package_doc/bfgsmin.html#bfgsmin
+pkg load optim
+
+% dimension of Laplace
+global N=20;
+
+% 1D Laplace with zero Dirichlet BC on both sides
+function M = get_m()
+  global N;
+  for i=1:N
+    M(i,i)=2;
+    if i>1
+      M(i,i-1)=-1;
+    end
+    if i<N
+      M(i,i+1)=-1;
+    end
+  end
+endfunction
+
+% RHS
+function b = get_b()
+  global N;
+  b(1:N,1)=1;
+endfunction
+
+global A=get_m();
+global b=get_b();
+
+x0(1:N,1)=1;
+
+% objective function with analytic gradient:
+% f(x) = 0.5 x*Ax - x*b
+% g(x) = Ax - b
+function [obj_value, gradient] = func(x)
+  global b;
+  global A;
+  Ax = A * x;
+  obj_value = 0.5 * (x' * Ax) - x'*b;
+  gradient  = Ax - b;
+endfunction
+
+mMax    = 4;      % maximum number of stored residuals
+itmax   = 100;    % maximum allowable number of iterations
+ftol    = 1e-12;  % function change tolerance
+xtol    = 1e-6;   % parameter change tolerance
+gtol    = 1e-5;   % gradient tolerance
+
+verb    = 2;      % verbosity [0,3]
+%
+% "used analytic gradient" prints 3 columns:
+% x_k  g_k  dx = alpha*p
+%
+
+control = {itmax;verb;1;1;mMax;ftol;xtol;gtol};
+[x, obj_value, convergence, iters] = bfgsmin("func", {x0}, control);
+
+fprintf("BFGS solution:\n");
+for i=1:N
+  fprintf("%d ", x(i))
+end
+fprintf("\n");
+
+xsol = A\b;
+
+fprintf("Exact solution:\n");
+for i=1:N
+  fprintf("%d ", xsol(i))
+end
+fprintf("\n");

diff --git a/tests/optimization/bfgs_04.output b/tests/optimization/bfgs_04.output
new file mode 100644 (file)
index 0000000..a43b1f2
--- /dev/null
+++ b/tests/optimization/bfgs_04.output
@@ -0,0 +1,240 @@
+
+DEAL:BFGS::Check 0     4.2426
+DEAL:BFGS::Starting value 4.2426
+------------------------------------------------
+Line search 0
+
+function value: -19.00000  stepsize: 9.00000
+
+          1.00000          0.00000         0.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000         -1.00000          9.00000
+          1.00000          0.00000         0.00000
+DEAL:BFGS::Check 1     17.493
+------------------------------------------------
+Line search 1
+
+function value: -100.00000  stepsize: 0.94444
+
+          1.00000         -9.00000          8.50000
+         10.00000          8.00000          8.50000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000         -1.00000         17.00000
+         10.00000          8.00000          8.50000
+          1.00000         -9.00000          8.50000
+DEAL:BFGS::Check 2     11.292
+------------------------------------------------
+Line search 2
+
+function value: -244.50000  stepsize: 0.58824
+
+          9.50000         -0.50000          2.50000
+         18.50000         -0.50000          2.50000
+         27.00000          7.50000         0.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000         -1.00000          5.00000
+         27.00000          7.50000         0.00000
+         18.50000         -0.50000          2.50000
+          9.50000         -0.50000          2.50000
+DEAL:BFGS::Check 3     7.7460
+------------------------------------------------
+Line search 3
+
+function value: -282.00000  stepsize: 1.20000
+
+         12.00000          2.00000         0.00000
+         21.00000          2.00000         0.00000
+         27.00000          0.00000         0.00000
+         32.00000          4.00000         0.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000         -1.00000          6.00000
+         32.00000          4.00000         0.00000
+         27.00000          0.00000         0.00000
+         21.00000          2.00000         0.00000
+         12.00000          2.00000         0.00000
+DEAL:BFGS::Check 4     9.1652
+------------------------------------------------
+Line search 4
+
+function value: -318.00000  stepsize: 0.87500
+
+         12.00000          2.00000         -1.75000
+         21.00000          2.00000         -1.75000
+         27.00000          0.00000         0.00000
+         32.00000         -2.00000          1.75000
+         38.00000          5.00000          1.75000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000         -1.00000          7.00000
+         38.00000          5.00000          1.75000
+         32.00000         -2.00000          1.75000
+         27.00000          0.00000         0.00000
+         21.00000          2.00000         -1.75000
+         12.00000          2.00000         -1.75000
+DEAL:BFGS::Check 5     6.6802
+------------------------------------------------
+Line search 5
+
+function value: -354.75000  stepsize: 0.64762
+
+         10.25000          0.25000         -0.85000
+         19.25000          0.25000         -0.85000
+         27.00000          0.00000         0.00000
+         33.75000         -0.25000          0.85000
+         39.75000         -0.25000          0.85000
+         45.00000          4.25000         0.00000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000         -1.00000          3.40000
+         45.00000          4.25000         0.00000
+         39.75000         -0.25000          0.85000
+         33.75000         -0.25000          0.85000
+         27.00000          0.00000         0.00000
+         19.25000          0.25000         -0.85000
+         10.25000          0.25000         -0.85000
+DEAL:BFGS::Check 6     4.5166
+------------------------------------------------
+Line search 6
+
+function value: -369.20000  stepsize: 0.88235
+
+          9.40000         -0.60000         0.00000
+         18.40000         -0.60000         0.00000
+         27.00000          0.00000         0.00000
+         34.60000          0.60000         0.00000
+         40.60000          0.60000         0.00000
+         45.00000          0.00000         0.00000
+         48.40000          2.40000         0.00000
+         48.40000         -1.00000          3.00000
+         48.40000         -1.00000          3.00000
+         48.40000         -1.00000          3.00000
+         48.40000         -1.00000          3.00000
+         48.40000         -1.00000          3.00000
+         48.40000         -1.00000          3.00000
+         48.40000          2.40000         0.00000
+         45.00000          0.00000         0.00000
+         40.60000          0.60000         0.00000
+         34.60000          0.60000         0.00000
+         27.00000          0.00000         0.00000
+         18.40000         -0.60000          0.00000
+          9.40000         -0.60000         0.00000
+DEAL:BFGS::Check 7     3.9497
+------------------------------------------------
+Line search 7
+
+function value: -378.20000  stepsize: 0.72222
+
+          9.40000         -0.60000          0.43333
+         18.40000         -0.60000          0.43333
+         27.00000          0.00000         0.00000
+         34.60000          0.60000         -0.43333
+         40.60000          0.60000         -0.43333
+         45.00000          0.00000         0.00000
+         48.40000         -0.60000          0.43333
+         51.40000          2.00000          0.43333
+         51.40000         -1.00000          2.60000
+         51.40000         -1.00000          2.60000
+         51.40000         -1.00000          2.60000
+         51.40000         -1.00000          2.60000
+         51.40000          2.00000          0.43333
+         48.40000         -0.60000          0.43333
+         45.00000          0.00000         0.00000
+         40.60000          0.60000         -0.43333
+         34.60000          0.60000         -0.43333
+         27.00000          0.00000         0.00000
+         18.40000         -0.60000          0.43333
+          9.40000         -0.60000          0.43333
+DEAL:BFGS::Check 8     2.2485
+------------------------------------------------
+Line search 8
+
+function value: -383.83333  stepsize: 0.46154
+
+          9.83333         -0.16667          0.16667
+         18.83333         -0.16667          0.16667
+         27.00000          0.00000         0.00000
+         34.16667          0.16667         -0.16667
+         40.16667          0.16667         -0.16667
+         45.00000          0.00000         0.00000
+         48.83333         -0.16667          0.16667
+         51.83333         -0.16667          0.16667
+         54.00000          1.16667         0.00000
+         54.00000         -1.00000          1.00000
+         54.00000         -1.00000          1.00000
+         54.00000          1.16667         0.00000
+         51.83333         -0.16667          0.16667
+         48.83333         -0.16667          0.16667
+         45.00000          0.00000         0.00000
+         40.16667          0.16667         -0.16667
+         34.16667          0.16667         -0.16667
+         27.00000          0.00000         0.00000
+         18.83333         -0.16667          0.16667
+          9.83333         -0.16667          0.16667
+DEAL:BFGS::Check 9     0.0000
+DEAL:BFGS::Convergence step 9 value 0.0000
+DEAL::Limited memory BFGS solution:
+DEAL::10.000 19.000 27.000 34.000 40.000 45.000 49.000 52.000 54.000 55.000 55.000 54.000 52.000 49.000 45.000 40.000 34.000 27.000 19.000 10.000 

diff --git a/tests/optimization/bfgs_04.output.octave b/tests/optimization/bfgs_04.output.octave
new file mode 100644 (file)
index 0000000..02af83e
--- /dev/null
+++ b/tests/optimization/bfgs_04.output.octave
@@ -0,0 +1,268 @@
+------------------------------------------------
+bfgsmin iteration 0  convergence (f g p): 0 0 0
+
+function value: -19  stepsize: 9  
+
+used analytic gradient
+        1.00000         0.00000        -0.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000        -1.00000         9.00000
+        1.00000         0.00000        -0.00000
+------------------------------------------------
+bfgsmin iteration 1  convergence (f g p): 0 0 0
+
+function value: -100  stepsize: 0.944444  
+
+used analytic gradient
+        1.00000        -9.00000         8.50000
+       10.00000         8.00000         8.50000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000        -1.00000        17.00000
+       10.00000         8.00000         8.50000
+        1.00000        -9.00000         8.50000
+------------------------------------------------
+bfgsmin iteration 2  convergence (f g p): 0 0 0
+
+function value: -244.5  stepsize: 0.588235  
+
+used analytic gradient
+        9.50000        -0.50000         2.50000
+       18.50000        -0.50000         2.50000
+       27.00000         7.50000        -0.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000        -1.00000         5.00000
+       27.00000         7.50000        -0.00000
+       18.50000        -0.50000         2.50000
+        9.50000        -0.50000         2.50000
+------------------------------------------------
+bfgsmin iteration 3  convergence (f g p): 0 0 0
+
+function value: -282  stepsize: 1.2  
+
+used analytic gradient
+       12.00000         2.00000        -0.00000
+       21.00000         2.00000        -0.00000
+       27.00000        -0.00000        -0.00000
+       32.00000         4.00000        -0.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000        -1.00000         6.00000
+       32.00000         4.00000        -0.00000
+       27.00000        -0.00000        -0.00000
+       21.00000         2.00000        -0.00000
+       12.00000         2.00000        -0.00000
+------------------------------------------------
+bfgsmin iteration 4  convergence (f g p): 0 0 0
+
+function value: -318  stepsize: 1.75  
+
+used analytic gradient
+       12.00000         2.00000        -1.75000
+       21.00000         2.00000        -1.75000
+       27.00000         0.00000         0.00000
+       32.00000        -2.00000         1.75000
+       38.00000         5.00000         1.75000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000        -1.00000         7.00000
+       38.00000         5.00000         1.75000
+       32.00000        -2.00000         1.75000
+       27.00000         0.00000         0.00000
+       21.00000         2.00000        -1.75000
+       12.00000         2.00000        -1.75000
+------------------------------------------------
+bfgsmin iteration 5  convergence (f g p): 0 0 0
+
+function value: -354.75  stepsize: 0.971429  
+
+used analytic gradient
+       10.25000         0.25000        -0.85000
+       19.25000         0.25000        -0.85000
+       27.00000        -0.00000        -0.00000
+       33.75000        -0.25000         0.85000
+       39.75000        -0.25000         0.85000
+       45.00000         4.25000        -0.00000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000        -1.00000         3.40000
+       45.00000         4.25000        -0.00000
+       39.75000        -0.25000         0.85000
+       33.75000        -0.25000         0.85000
+       27.00000        -0.00000        -0.00000
+       19.25000         0.25000        -0.85000
+       10.25000         0.25000        -0.85000
+------------------------------------------------
+bfgsmin iteration 6  convergence (f g p): 0 0 0
+
+function value: -369.2  stepsize: 2.20588  
+
+used analytic gradient
+        9.40000        -0.60000         0.00000
+       18.40000        -0.60000         0.00000
+       27.00000        -0.00000        -0.00000
+       34.60000         0.60000        -0.00000
+       40.60000         0.60000        -0.00000
+       45.00000        -0.00000        -0.00000
+       48.40000         2.40000        -0.00000
+       48.40000        -1.00000         3.00000
+       48.40000        -1.00000         3.00000
+       48.40000        -1.00000         3.00000
+       48.40000        -1.00000         3.00000
+       48.40000        -1.00000         3.00000
+       48.40000        -1.00000         3.00000
+       48.40000         2.40000        -0.00000
+       45.00000        -0.00000        -0.00000
+       40.60000         0.60000        -0.00000
+       34.60000         0.60000        -0.00000
+       27.00000        -0.00000        -0.00000
+       18.40000        -0.60000         0.00000
+        9.40000        -0.60000         0.00000
+------------------------------------------------
+bfgsmin iteration 7  convergence (f g p): 0 0 0
+
+function value: -378.2  stepsize: 1.44444  
+
+used analytic gradient
+        9.40000        -0.60000         0.43333
+       18.40000        -0.60000         0.43333
+       27.00000        -0.00000        -0.00000
+       34.60000         0.60000        -0.43333
+       40.60000         0.60000        -0.43333
+       45.00000         0.00000         0.00000
+       48.40000        -0.60000         0.43333
+       51.40000         2.00000         0.43333
+       51.40000        -1.00000         2.60000
+       51.40000        -1.00000         2.60000
+       51.40000        -1.00000         2.60000
+       51.40000        -1.00000         2.60000
+       51.40000         2.00000         0.43333
+       48.40000        -0.60000         0.43333
+       45.00000         0.00000         0.00000
+       40.60000         0.60000        -0.43333
+       34.60000         0.60000        -0.43333
+       27.00000        -0.00000        -0.00000
+       18.40000        -0.60000         0.43333
+        9.40000        -0.60000         0.43333
+------------------------------------------------
+bfgsmin iteration 8  convergence (f g p): 0 0 0
+
+function value: -383.833  stepsize: 0.807692  
+
+used analytic gradient
+        9.83333        -0.16667         0.16667
+       18.83333        -0.16667         0.16667
+       27.00000         0.00000        -0.00000
+       34.16667         0.16667        -0.16667
+       40.16667         0.16667        -0.16667
+       45.00000        -0.00000        -0.00000
+       48.83333        -0.16667         0.16667
+       51.83333        -0.16667         0.16667
+       54.00000         1.16667         0.00000
+       54.00000        -1.00000         1.00000
+       54.00000        -1.00000         1.00000
+       54.00000         1.16667         0.00000
+       51.83333        -0.16667         0.16667
+       48.83333        -0.16667         0.16667
+       45.00000        -0.00000        -0.00000
+       40.16667         0.16667        -0.16667
+       34.16667         0.16667        -0.16667
+       27.00000         0.00000        -0.00000
+       18.83333        -0.16667         0.16667
+        9.83333        -0.16667         0.16667
+------------------------------------------------
+bfgsmin final results: 9 iterations
+
+function value: -385
+
+STRONG CONVERGENCE
+Function conv 1  Param conv 1  Gradient conv 1
+
+used analytic gradient
+          param    gradient (n)          change
+       10.00000        -0.00000         0.00000
+       19.00000        -0.00000         0.00000
+       27.00000        -0.00000         0.00000
+       34.00000         0.00000        -0.00000
+       40.00000         0.00000        -0.00000
+       45.00000        -0.00000        -0.00000
+       49.00000         0.00000         0.00000
+       52.00000        -0.00000         0.00000
+       54.00000         0.00000         0.00000
+       55.00000        -0.00000         0.00000
+       55.00000        -0.00000         0.00000
+       54.00000         0.00000         0.00000
+       52.00000        -0.00000         0.00000
+       49.00000         0.00000         0.00000
+       45.00000        -0.00000        -0.00000
+       40.00000         0.00000        -0.00000
+       34.00000         0.00000        -0.00000
+       27.00000        -0.00000         0.00000
+       19.00000        -0.00000         0.00000
+       10.00000        -0.00000         0.00000
+BFGS solution:
+10 19 27 34 40 45 49 52 54 55 55 54 52 49 45 40 34 27 19 10 
+Exact solution:
+10 19 27 34 40 45 49 52 54 55 55 54 52 49 45 40 34 27 19 10 

diff --git a/tests/optimization/bfgs_05.cc b/tests/optimization/bfgs_05.cc
new file mode 100644 (file)
index 0000000..81d51ca
--- /dev/null
+++ b/tests/optimization/bfgs_05.cc
@@ -0,0 +1,237 @@
+//-----------------------------------------------------------
+//
+//    Copyright (C) 2018 by the deal.II authors
+//
+//    This file is part of the deal.II library.
+//
+//    The deal.II library is free software; you can use it, redistribute
+//    it, and/or modify it under the terms of the GNU Lesser General
+//    Public License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//    The full text of the license can be found in the file LICENSE.md at
+//    the top level directory of deal.II.
+//
+//---------------------------------------------------------------
+
+// test limited memory BFGS with Rosenbrock function
+// Octave with optim@1.5.2 converges with:
+//
+// 114 iterations
+// function value: 6.74546e-13
+// linf_norm =     1.4103e-06
+//
+// whereas we converge with
+//
+// 127 iterations
+// function value: 1.3096e-13
+// linf_norm =     1.6564e-07
+// function calls: 133
+//
+// Scipy@1.1.0 converges with
+//
+// 130 iterations
+// function value: 9.10208322706e-13
+// linf_norm =     1.1038165888e-06
+// Gradient norm:  1.72580704033e-05
+// function calls: 143
+//
+// Note that both Octave and Scipy can not be matched exactly as
+// there is no simple way to control all parameters and algorithms (i.e. line
+// searches)
+
+
+#include <deal.II/base/logstream.h>
+
+#include <deal.II/lac/full_matrix.h>
+#include <deal.II/lac/vector.h>
+
+#include <deal.II/optimization/line_minimization.h>
+#include <deal.II/optimization/solver_bfgs.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "../tests.h"
+
+using namespace LineMinimization;
+
+template <typename number>
+void
+test()
+{
+  auto &out = deallog.get_file_stream();
+  out << std::setprecision(5) << std::fixed << std::right;
+
+  typedef Vector<number> VectorType;
+
+  // size of the problem
+  const unsigned int N = 21;
+
+  // parameters:
+  const unsigned int itmax = 150;
+  const double       gtol  = 1e-5; // gradient tolerance
+  const unsigned int m_max = 3;
+
+  // solution
+  VectorType x(N), x_shifted(x);
+  x = 0.;
+
+  // shift minimizer to this point
+  VectorType location(x);
+  for (unsigned int i = 0; i < N; ++i)
+    location(i) = double(i) / (N - 1);
+
+  // see
+  // https://sourceforge.net/p/octave/optim/ci/default/tree/inst/rosenbrock.m#l26
+  const auto rosenbrok = [&](VectorType &x, VectorType &g) {
+    const unsigned int N   = x.size();
+    double             res = 0.;
+    g                      = 0;
+    for (unsigned int i = 0; i < N; ++i)
+      {
+        const double xi2 = x(i) * x(i);
+
+        if (i < N - 1)
+          {
+            res += 100. * dealii::Utilities::fixed_power<2>(x(i + 1) - xi2) +
+                   dealii::Utilities::fixed_power<2>(1. - x(i));
+
+            g(i) += -400. * x(i) * (x(i + 1) - xi2) - 2. * (1. - x(i));
+          }
+
+        if (i > 0)
+          g(i) += 200. * (x(i) - x(i - 1) * x(i - 1));
+      }
+    return res;
+  };
+
+
+
+  unsigned int tot_fun_calls = 0;
+  const auto   func          = [&](const VectorType &x, VectorType &g) {
+    tot_fun_calls++;
+    for (unsigned int i = 0; i < x.size(); ++i)
+      x_shifted(i) = x(i) - location(i) + 1.;
+
+    return rosenbrok(x_shifted, g);
+  };
+
+  VectorType x0(x), dx(x), g_old(x);
+
+  bool   first_step = true;
+  double f_prev     = 0.;
+
+  const unsigned int print_n_iterations     = 5;
+  unsigned int       iteration              = 0;
+  unsigned int       line_search_iterations = 0;
+  const auto         line_min               = [&](number &          f,
+                            VectorType &      x,
+                            VectorType &      g,
+                            const VectorType &p) {
+    if (iteration <= print_n_iterations)
+      out << "------------------------------------------------" << std::endl
+          << "Line search " << iteration << std::endl
+          << std::endl;
+
+    // save current solution value and gradient
+    x0    = x;
+    g_old = g;
+
+    const double f0 = f;
+    const double g0 = g * p;
+    Assert(g0 < 0, ExcInternalError());
+
+    // see scipy implementation
+    // https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.line_search.html#scipy.optimize.line_search
+    // and Eq. 2.6.8 in Fletcher 2013, Practical methods of optimization
+    double df = f_prev - f;
+    Assert(first_step || df >= 0., ExcInternalError());
+    df = std::max(df, 100. * std::numeric_limits<double>::epsilon());
+    const double a1 = (first_step ? 1. : std::min(1., -1.01 * 2. * df / g0));
+    Assert(a1 > 0., ExcInternalError());
+    f_prev = f;
+
+    // 1D line-search function
+    auto line_func = [&](const double &x_line) -> std::pair<double, double> {
+      x = x0;
+      x.add(x_line, p);
+      const double f_line = func(x, g);
+      const double g_line = g * p;
+      f                   = f_line;
+
+      return std::make_pair(f_line, g_line);
+    };
+
+    const auto res =
+      line_search<double>(line_func, f0, g0, poly_fit<double>, a1, 0.9, 0.001);
+
+    line_search_iterations += res.second;
+
+    if (iteration <= print_n_iterations)
+      {
+        out << "function value: " << f0 << "  stepsize: " << res.first
+            << std::endl
+            << std::endl;
+
+        // change:
+        dx = p;
+        dx *= res.first;
+
+        const std::string s = "        ";
+        for (unsigned int i = 0; i < N; ++i)
+          out << s << std::setw(9) << x0(i) << s << std::setw(9) << g_old(i)
+              << s << std::setw(9) << dx(i) << std::endl;
+      }
+
+    if (first_step)
+      first_step = false;
+
+    iteration++;
+    return res.first;
+  };
+
+  const auto preconditioner = [](VectorType &                         g,
+                                 const FiniteSizeHistory<VectorType> &s,
+                                 const FiniteSizeHistory<VectorType> &y) {
+    if (s.size() > 0)
+      {
+        // default preconditioning using the oldest {s,y} pair, see
+        // lbfgs_recursion() in __bfgsmin.cc of "optim" Octave package.
+        const unsigned int i  = s.size() - 1;
+        const double       yy = y[i] * y[i];
+        const double       sy = s[i] * y[i];
+        Assert(yy > 0 && sy > 0, ExcInternalError());
+        g *= sy / yy;
+      }
+  };
+
+  SolverControl solver_control(itmax, gtol, false);
+  typename SolverBFGS<VectorType>::AdditionalData data(m_max, false);
+  SolverBFGS<VectorType>                          solver(solver_control, data);
+  solver.connect_line_search_slot(line_min);
+  solver.connect_preconditioner_slot(preconditioner);
+  solver.solve(func, x);
+
+  Assert(tot_fun_calls == line_search_iterations + 1, ExcInternalError());
+
+  deallog << "Limited memory BFGS solution:" << std::endl;
+  x.print(deallog);
+
+  deallog << "Function value: " << func(x, x0) << std::endl;
+
+  x.add(-1, location);
+  deallog << "Linf error in solution: " << x.linfty_norm() << std::endl;
+
+  deallog << "function calls: "
+          << (tot_fun_calls - 1) /*one evaluation above*/ << std::endl;
+}
+
+int
+main()
+{
+  std::ofstream logfile("output");
+  deallog << std::setprecision(5);
+  deallog.attach(logfile);
+
+  test<double>();
+}

diff --git a/tests/optimization/bfgs_05.m b/tests/optimization/bfgs_05.m
new file mode 100755 (executable)
index 0000000..9114fdd
--- /dev/null
+++ b/tests/optimization/bfgs_05.m
@@ -0,0 +1,48 @@
+% //-----------------------------------------------------------
+% //
+% //    Copyright (C) 2018 by the deal.II authors
+% //
+% //    This file is part of the deal.II library.
+% //
+% //    The deal.II library is free software; you can use it, redistribute
+% //    it, and/or modify it under the terms of the GNU Lesser General
+% //    Public License as published by the Free Software Foundation; either
+% //    version 2.1 of the License, or (at your option) any later version.
+% //    The full text of the license can be found in the file LICENSE.md at
+% //    the top level directory of deal.II.
+% //
+% //---------------------------------------------------------------
+
+% a companion to bfgs_05.cc which minimizes the Rosenbrok function using Octave.
+
+% This file uses bfgsmin() from optim package: https://octave.sourceforge.io/optim/
+% https://octave.sourceforge.io/optim/package_doc/bfgsmin.html#bfgsmin
+%
+% take Example 3 from bfgsmin_example.m
+pkg load optim
+
+function [obj_value, gradient] = objective(theta, location)
+       x = theta - location + ones(rows(theta),1); % move minimizer to "location"
+       [obj_value, gradient] = rosenbrock(x);
+endfunction
+
+% problem parameters
+dim = 20; % dimension of Rosenbrock function
+theta0 = zeros(dim+1,1);  % starting values
+location = (0:dim)/dim;   % true values
+location = location';
+
+% solver parameers
+mMax    = 3;      % maximum number of stored residuals
+itmax   = 120;    % maximum allowable number of iterations
+ftol    = 1;      % function change tolerance
+xtol    = 1;      % parameter change tolerance
+gtol    = 1e-5;   % gradient tolerance
+
+verb    = 2;      % verbosity [0,3]
+
+control = {itmax;verb;1;1;mMax;ftol;xtol;gtol};
+
+[theta, obj_value, convergence] = bfgsmin("objective", {theta0, location}, control);
+
+linf_norm = norm(theta-location, 'inf')

diff --git a/tests/optimization/bfgs_05.output b/tests/optimization/bfgs_05.output
new file mode 100644 (file)
index 0000000..5873443
--- /dev/null
+++ b/tests/optimization/bfgs_05.output
@@ -0,0 +1,164 @@
+
+DEAL:BFGS::Starting value 63.201
+------------------------------------------------
+Line search 0
+
+function value: 44.59125  stepsize: 0.00688
+
+          0.00000         20.00000         -0.13754
+          0.00000         -9.15000          0.06292
+          0.00000        -15.10000          0.10384
+          0.00000        -18.65000          0.12825
+          0.00000        -20.10000          0.13822
+          0.00000        -19.75000          0.13582
+          0.00000        -17.90000          0.12309
+          0.00000        -14.85000          0.10212
+          0.00000        -10.90000          0.07496
+          0.00000         -6.35000          0.04367
+          0.00000         -1.50000          0.01032
+          0.00000          3.35000         -0.02304
+          0.00000          7.90000         -0.05433
+          0.00000         11.85000         -0.08149
+          0.00000         14.90000         -0.10246
+          0.00000         16.75000         -0.11519
+          0.00000         17.10000         -0.11759
+          0.00000         15.65000         -0.10762
+          0.00000         12.10000         -0.08321
+          0.00000          6.15000         -0.04229
+          0.00000         -0.50000          0.00344
+------------------------------------------------
+Line search 1
+
+function value: 31.46982  stepsize: 1.00000
+
+         -0.13754        -93.10302          0.04478
+          0.06292         62.82505         -0.04890
+          0.10384          7.39497          0.05454
+          0.12825          1.40648          0.07821
+          0.13822         -5.95844          0.09479
+          0.13582        -12.73220          0.10280
+          0.12309        -17.24202          0.10118
+          0.10212        -18.44290          0.08975
+          0.07496        -16.08048          0.06945
+          0.04367        -10.68239          0.04231
+          0.01032         -3.39997          0.01123
+         -0.02304          4.25994         -0.02039
+         -0.05433         10.76168         -0.04908
+         -0.08149         14.86845         -0.07183
+         -0.10246         15.91366         -0.08642
+         -0.11519         13.96456         -0.09163
+         -0.11759          9.83821         -0.08734
+         -0.10762          4.94807         -0.07424
+         -0.08321          0.98272         -0.05341
+         -0.04229         -0.50983         -0.02573
+          0.00344          0.67580          0.00120
+------------------------------------------------
+Line search 2
+
+function value: 21.31066  stepsize: 1.00000
+
+         -0.09276        -51.32933          0.10125
+          0.01402        -21.64466          0.01449
+          0.15838         52.89125         -0.07330
+          0.20646         22.49149         -0.02224
+          0.23302         15.88337         -0.00831
+          0.23862          5.27077          0.01028
+          0.22428         -6.18067          0.02825
+          0.19187        -14.80591          0.03981
+          0.14441        -17.74562          0.04070
+          0.08597        -14.14528          0.03002
+          0.02154         -5.61538          0.01090
+         -0.04342          4.37621         -0.01060
+         -0.10341         11.95369         -0.02776
+         -0.15332         14.43652         -0.03582
+         -0.18888         11.33937         -0.03357
+         -0.20682          4.37039         -0.02343
+         -0.20494         -3.43285         -0.01016
+         -0.18186         -8.92041          0.00109
+         -0.13662         -9.88283          0.00651
+         -0.06802         -5.87687          0.00501
+          0.00464          0.86286         -0.00124
+------------------------------------------------
+Line search 3
+
+function value: 14.22694  stepsize: 1.00000
+
+          0.00849         15.56485          0.01765
+          0.02851        -18.55220          0.02206
+          0.08508        -19.66673          0.01341
+          0.18422         31.41559         -0.05258
+          0.22471         12.02393         -0.01608
+          0.24890          7.86342         -0.00314
+          0.25253          0.61442          0.01378
+          0.23168         -8.09261          0.03039
+          0.18510        -14.46074          0.03949
+          0.11600        -14.42561          0.03454
+          0.03244         -7.11434          0.01527
+         -0.05402          3.54421         -0.01005
+         -0.13117         11.41788         -0.02947
+         -0.18914         12.71412         -0.03534
+         -0.22245          8.06446         -0.02816
+         -0.23025          0.90841         -0.01401
+         -0.21510         -5.31486          0.00025
+         -0.18077         -8.64248          0.00988
+         -0.13010         -8.43792          0.01255
+         -0.06301         -4.79304          0.00785
+          0.00340          0.64542         -0.00140
+------------------------------------------------
+Line search 4
+
+function value: 11.33438  stepsize: 1.00000
+
+          0.02614         21.55972         -0.01245
+          0.05057         -9.41840          0.02192
+          0.09849          5.59813          0.00481
+          0.13164        -20.78185          0.00668
+          0.20863         17.72360         -0.03316
+          0.24576          5.72622         -0.00085
+          0.26631          3.26875          0.01611
+          0.26207         -1.85931          0.03631
+          0.22460         -8.59387          0.05269
+          0.15054        -12.44310          0.05270
+          0.04771         -8.34089          0.02750
+         -0.06407          2.00137         -0.01337
+         -0.16064         10.03092         -0.04540
+         -0.22449         10.20601         -0.05269
+         -0.25061          4.76104         -0.03931
+         -0.24426         -1.29442         -0.01853
+         -0.21485         -5.13072         -0.00047
+         -0.17089         -6.41353          0.01063
+         -0.11756         -5.67239          0.01374
+         -0.05517         -3.10137          0.00868
+          0.00199          0.39356         -0.00167
+------------------------------------------------
+Line search 5
+
+function value: 9.39817  stepsize: 1.00000
+
+          0.01369          2.09034         -0.00803
+          0.07249         16.27957         -0.01397
+          0.10330         -1.08753          0.00313
+          0.13832         -3.17010          0.00613
+          0.17547        -17.22664          0.01211
+          0.24491         11.61622         -0.01544
+          0.28242          5.06870         -0.00275
+          0.29838          4.94126          0.00290
+          0.27729          1.08018          0.01239
+          0.20325         -6.13174          0.02175
+          0.07521         -8.87643          0.01861
+         -0.07744         -1.13673         -0.00204
+         -0.20604          6.68937         -0.02043
+         -0.27718          5.04666         -0.02017
+         -0.28992         -0.97195         -0.00884
+         -0.26279         -4.68297          0.00148
+         -0.21531         -5.19031          0.00692
+         -0.16026         -4.13583          0.00845
+         -0.10382         -2.78767          0.00745
+         -0.04649         -1.29372          0.00407
+          0.00033          0.06337         -0.00053
+DEAL:BFGS::Convergence step 127 value 8.1397e-06
+DEAL::Limited memory BFGS solution:
+DEAL::-4.4833e-09 0.050000 0.10000 0.15000 0.20000 0.25000 0.30000 0.35000 0.40000 0.45000 0.50000 0.55000 0.60000 0.65000 0.70000 0.75000 0.80000 0.85000 0.90000 0.95000 1.0000 
+DEAL::Function value: 1.3096e-13
+DEAL::Linf error in solution: 1.6564e-07
+DEAL::function calls: 133

diff --git a/tests/optimization/bfgs_05.output.octave b/tests/optimization/bfgs_05.output.octave
new file mode 100644 (file)
index 0000000..b83a7e0
--- /dev/null
+++ b/tests/optimization/bfgs_05.output.octave
@@ -0,0 +1,3110 @@
+------------------------------------------------
+bfgsmin iteration 0  convergence (f g p): 0 0 0
+
+function value: 44.5913  stepsize: 0.00711092  
+
+used analytic gradient
+        0.00000        20.00000        -0.14222
+        0.00000        -9.15000         0.06506
+        0.00000       -15.10000         0.10737
+        0.00000       -18.65000         0.13262
+        0.00000       -20.10000         0.14293
+        0.00000       -19.75000         0.14044
+        0.00000       -17.90000         0.12729
+        0.00000       -14.85000         0.10560
+        0.00000       -10.90000         0.07751
+        0.00000        -6.35000         0.04515
+        0.00000        -1.50000         0.01067
+        0.00000         3.35000        -0.02382
+        0.00000         7.90000        -0.05618
+        0.00000        11.85000        -0.08426
+        0.00000        14.90000        -0.10595
+        0.00000        16.75000        -0.11911
+        0.00000        17.10000        -0.12160
+        0.00000        15.65000        -0.11129
+        0.00000        12.10000        -0.08604
+        0.00000         6.15000        -0.04373
+        0.00000        -0.50000         0.00356
+------------------------------------------------
+bfgsmin iteration 1  convergence (f g p): 0 0 0
+
+function value: 31.5771  stepsize: 0.00165409  
+
+used analytic gradient
+       -0.14222       -96.10749         0.03592
+        0.06506        65.21654        -0.04736
+        0.10737         8.38754         0.06630
+        0.13262         2.41493         0.09345
+        0.14293        -5.13573         0.11198
+        0.14044       -12.20696         0.12046
+        0.12729       -17.02499         0.11785
+        0.10560       -18.46283         0.10409
+        0.07751       -16.22190         0.08029
+        0.04515       -10.82911         0.04878
+        0.01067        -3.47246         0.01285
+       -0.02382         4.28229        -0.02365
+       -0.05618        10.83832        -0.05685
+       -0.08426        14.91554        -0.08333
+       -0.10595        15.83746        -0.10053
+       -0.11911        13.70140        -0.10703
+       -0.12160         9.38956        -0.10258
+       -0.11129         4.39789        -0.08777
+       -0.08604         0.48582        -0.06362
+       -0.04373        -0.78168        -0.03084
+        0.00356         0.70324         0.00158
+------------------------------------------------
+bfgsmin iteration 2  convergence (f g p): 0 0 0
+
+function value: 23.1595  stepsize: 0.00158028  
+
+used analytic gradient
+       -0.10630       -60.62528         0.02658
+        0.01770       -19.38494         0.04778
+        0.17367        60.53681        -0.04140
+        0.22607        29.15174         0.00564
+        0.25491        22.52286         0.01687
+        0.26090        10.67866         0.03060
+        0.24514        -2.76216         0.04305
+        0.20968       -13.44473         0.04948
+        0.15780       -17.85744         0.04621
+        0.09393       -14.81813         0.03239
+        0.02352        -6.07835         0.01084
+       -0.04747         4.37845        -0.01284
+       -0.11303        12.07714        -0.03245
+       -0.16759        13.98041        -0.04355
+       -0.20648         9.67143        -0.04499
+       -0.22614         1.30606        -0.03878
+       -0.22417        -7.47705        -0.02876
+       -0.19906       -13.04828        -0.01874
+       -0.14966       -13.04999        -0.01091
+       -0.07457        -7.41192        -0.00461
+        0.00513         0.90554         0.00002
+------------------------------------------------
+bfgsmin iteration 3  convergence (f g p): 1 0 1
+
+function value: 19.6471  stepsize: 0.435579  
+
+used analytic gradient
+       -0.07972       -62.20698         0.04419
+        0.06548        33.30355        -0.01009
+        0.13228        -6.37137        -0.03239
+        0.23171        45.92197        -0.07369
+        0.27178        26.44966        -0.06852
+        0.29150        18.85397        -0.06567
+        0.28818         7.28696        -0.05562
+        0.25917        -5.43594        -0.04052
+        0.20400       -14.52432        -0.02409
+        0.12632       -15.48456        -0.01014
+        0.03436        -7.80189        -0.00015
+       -0.06032         3.60477         0.00766
+       -0.14548        11.50157         0.01649
+       -0.21114        11.46823         0.02787
+       -0.25147         4.28266         0.04032
+       -0.26492        -5.80243         0.05044
+       -0.25294       -14.23225         0.05492
+       -0.21780       -18.02237         0.05169
+       -0.16057       -15.95030         0.03999
+       -0.07918        -8.57877         0.02017
+        0.00516         0.86076        -0.00145
+------------------------------------------------
+bfgsmin iteration 4  convergence (f g p): 1 0 1
+
+function value: 13.7213  stepsize: 1.14318  
+
+used analytic gradient
+       -0.03553       -29.07749         0.06636
+        0.05539        19.44095         0.00506
+        0.09989        -5.47804         0.00191
+        0.15802         6.83697         0.00521
+        0.20327         9.76823         0.00791
+        0.22582         1.48244         0.04141
+        0.23257        -3.68271         0.06391
+        0.21865        -9.27686         0.08193
+        0.17991       -13.43998         0.08726
+        0.11618       -13.17660         0.07068
+        0.03421        -6.77333         0.02979
+       -0.05265         3.20740        -0.02316
+       -0.12899        11.05492        -0.06634
+       -0.18327        12.91806        -0.08478
+       -0.21114         9.40742        -0.07908
+       -0.21448         3.58559        -0.05945
+       -0.19802        -1.83960        -0.03651
+       -0.16610        -5.38820        -0.01713
+       -0.12058        -6.28669        -0.00471
+       -0.05901        -3.89232         0.00019
+        0.00370         0.72416        -0.00135
+------------------------------------------------
+bfgsmin iteration 5  convergence (f g p): 1 0 1
+
+function value: 10.5274  stepsize: 1.28732  
+
+used analytic gradient
+        0.03083        21.56713        -0.01910
+        0.06046        -2.64280        -0.00410
+        0.10180        -7.69289        -0.00259
+        0.16323         8.21389        -0.02205
+        0.21117        -0.95188        -0.02318
+        0.26723        14.56577        -0.03991
+        0.29648         9.22979        -0.03415
+        0.30059         5.28404        -0.02748
+        0.26717        -2.31301        -0.01392
+        0.18686        -9.74861         0.00152
+        0.06400        -9.31879         0.00860
+       -0.07581         0.28861         0.00479
+       -0.19533         8.06118         0.00227
+       -0.26805         6.07457         0.00981
+       -0.29022        -1.27328         0.02182
+       -0.27393        -7.18899         0.03007
+       -0.23453        -9.56499         0.03214
+       -0.18324        -9.30319         0.02930
+       -0.12529        -7.42442         0.02274
+       -0.05883        -3.90292         0.01171
+        0.00235         0.45489        -0.00116
+------------------------------------------------
+bfgsmin iteration 6  convergence (f g p): 1 0 1
+
+function value: 9.16809  stepsize: 0.979367  
+
+used analytic gradient
+        0.01173         7.00016        -0.00461
+        0.05636         2.01369        -0.00228
+        0.09922         0.19366        -0.00537
+        0.14117        -3.67529         0.00929
+        0.18799         0.61248        -0.00447
+        0.22732        -2.60715         0.01560
+        0.26233         2.50117         0.00433
+        0.27311        -1.16069         0.02123
+        0.25325        -3.60103         0.03070
+        0.18838        -6.54700         0.03651
+        0.07261        -7.07714         0.02927
+       -0.07103        -0.62232        -0.00292
+       -0.19306         7.02795        -0.03688
+       -0.25824         7.11943        -0.04102
+       -0.26840         2.63406        -0.02492
+       -0.24386        -0.95432        -0.00848
+       -0.20239        -2.49441         0.00202
+       -0.15393        -2.79980         0.00739
+       -0.10254        -2.51403         0.00880
+       -0.04711        -1.41931         0.00549
+        0.00119         0.23712        -0.00111
+------------------------------------------------
+bfgsmin iteration 7  convergence (f g p): 1 0 1
+
+function value: 8.52604  stepsize: 0.439945  
+
+used analytic gradient
+        0.00712         4.12561        -0.00651
+        0.05408         3.72285        -0.00717
+        0.09384        -7.94163         0.00410
+        0.15046         9.51407        -0.00524
+        0.18352       -13.59227         0.00960
+        0.24292        12.74922        -0.00777
+        0.26666        -7.74448         0.00554
+        0.29434         4.82690        -0.00127
+        0.28395         0.49390         0.00632
+        0.22489        -2.18759         0.01219
+        0.10188        -3.83202         0.01390
+       -0.07395        -2.79881         0.00351
+       -0.22994         2.58590        -0.01323
+       -0.29926         2.38129        -0.01366
+       -0.29332        -1.15975        -0.00385
+       -0.25235        -2.48362         0.00296
+       -0.20037        -2.07616         0.00542
+       -0.14654        -1.31041         0.00568
+       -0.09375        -0.75989         0.00494
+       -0.04162        -0.31482         0.00272
+        0.00008         0.00241        -0.00040
+------------------------------------------------
+bfgsmin iteration 8  convergence (f g p): 1 0 1
+
+function value: 8.23001  stepsize: 0.906195  
+
+used analytic gradient
+        0.00061         1.72333        -0.00327
+        0.04691        -2.50689         0.00287
+        0.09794         1.08557        -0.00166
+        0.14523        -1.19777         0.00064
+        0.19312         0.96755         0.00019
+        0.23515        -0.91542        -0.00085
+        0.27220         1.09461        -0.00077
+        0.29307        -0.86920         0.00023
+        0.29027         1.92007        -0.00418
+        0.23708        -0.35894         0.00153
+        0.11578        -1.60121         0.00508
+       -0.07045        -3.10674         0.00765
+       -0.24317         0.08524        -0.00127
+       -0.31292         0.54926        -0.00216
+       -0.29718        -1.70534         0.00404
+       -0.24939        -1.88015         0.00517
+       -0.19495        -0.99916         0.00340
+       -0.14086        -0.19968         0.00155
+       -0.08880         0.19614         0.00048
+       -0.03890         0.22007         0.00000
+       -0.00032        -0.08854         0.00013
+------------------------------------------------
+bfgsmin iteration 9  convergence (f g p): 1 0 1
+
+function value: 8.19127  stepsize: 1.08858  
+
+used analytic gradient
+       -0.00267        -2.04125         0.00076
+        0.04978         2.32889        -0.00196
+        0.09628        -1.97322         0.00228
+        0.14587         0.02692         0.00008
+        0.19331         1.23871        -0.00152
+        0.23431        -1.51432         0.00128
+        0.27143         0.60101        -0.00155
+        0.29330         1.21446        -0.00241
+        0.28609        -2.19106         0.00001
+        0.23861         0.59230        -0.00023
+        0.12086        -1.39499         0.00601
+       -0.06280        -1.75263         0.00889
+       -0.24444        -1.22880         0.00144
+       -0.31508         0.13546        -0.00141
+       -0.29314        -0.87320         0.00495
+       -0.24422        -0.86169         0.00567
+       -0.19156        -0.33678         0.00331
+       -0.13931         0.09516         0.00107
+       -0.08832         0.28502        -0.00011
+       -0.03889         0.21816        -0.00038
+       -0.00019        -0.06233         0.00021
+------------------------------------------------
+bfgsmin iteration 10  convergence (f g p): 1 0 1
+
+function value: 8.16392  stepsize: 5.68739  
+
+used analytic gradient
+       -0.00190        -0.65044         0.00426
+        0.04781        -0.84794         0.00237
+        0.09856         1.04814        -0.00243
+        0.14595        -0.19334         0.00131
+        0.19179        -0.80624         0.00111
+        0.23559         0.94901        -0.00222
+        0.26988        -0.44522        -0.00021
+        0.29089        -0.39024        -0.00092
+        0.28611        -1.19107         0.00434
+        0.23838        -1.46836         0.00487
+        0.12688        -0.45894         0.00728
+       -0.05391        -0.60443         0.01234
+       -0.24300        -2.21862         0.01274
+       -0.31649        -0.30383         0.00124
+       -0.28819         0.10806         0.00559
+       -0.23856         0.23049         0.00512
+       -0.18825         0.29277         0.00190
+       -0.13824         0.29457        -0.00068
+       -0.08843         0.25977        -0.00179
+       -0.03927         0.14185        -0.00134
+        0.00002        -0.01826         0.00036
+------------------------------------------------
+bfgsmin iteration 11  convergence (f g p): 1 0 1
+
+function value: 8.13367  stepsize: 2.97746  
+
+used analytic gradient
+        0.00235         1.82238         0.00312
+        0.05018         0.78808         0.00196
+        0.09612        -2.84700         0.00398
+        0.14726         1.64514        -0.00244
+        0.19290         0.65006         0.00016
+        0.23336        -1.58261         0.00060
+        0.26967         0.59063        -0.00205
+        0.28996        -2.78055         0.00666
+        0.29045         1.03772         0.00579
+        0.24325        -1.90091         0.01479
+        0.13415        -1.34231         0.01692
+       -0.04157        -0.40629         0.02340
+       -0.23025        -1.47743         0.02909
+       -0.31525        -0.96746         0.00573
+       -0.28260         1.15977         0.00647
+       -0.23344         1.19655         0.00523
+       -0.18635         0.64247         0.00104
+       -0.13891         0.15941        -0.00209
+       -0.09023        -0.08664        -0.00329
+       -0.04060        -0.12238        -0.00221
+        0.00039         0.05985         0.00049
+------------------------------------------------
+bfgsmin iteration 12  convergence (f g p): 1 0 1
+
+function value: 8.07513  stepsize: 3.9278  
+
+used analytic gradient
+        0.00547         3.56653        -0.00698
+        0.05214        -0.09011         0.00616
+        0.10010         1.31771         0.00670
+        0.14482        -2.43966         0.00360
+        0.19307         1.54817        -0.00693
+        0.23396        -0.25721         0.00342
+        0.26762        -4.17441         0.01296
+        0.29662         1.90348         0.01198
+        0.29624        -1.89582         0.02679
+        0.25803         1.17141         0.04036
+        0.15107        -3.13403         0.06704
+       -0.01817        -1.68458         0.08023
+       -0.20116         0.79498         0.08655
+       -0.30952        -2.08981         0.02763
+       -0.27613         2.29093         0.00920
+       -0.22821         2.16400         0.00591
+       -0.18531         0.82122        -0.00072
+       -0.14100        -0.24853        -0.00537
+       -0.09351        -0.72421        -0.00682
+       -0.04281        -0.55917        -0.00424
+        0.00088         0.16597         0.00075
+------------------------------------------------
+bfgsmin iteration 13  convergence (f g p): 1 0 1
+
+function value: 7.89915  stepsize: 1  
+
+used analytic gradient
+       -0.00150        -4.51716        -0.00082
+        0.05830         6.25712        -0.00289
+        0.10680         4.16889         0.00013
+        0.14842         1.23070         0.00119
+        0.18613        -8.05716         0.00371
+        0.23738         0.72482         0.00181
+        0.28058         2.24598         0.00679
+        0.30860        -2.24915         0.01032
+        0.32303         1.99440         0.01537
+        0.29840        -0.45542         0.02660
+        0.21811         0.14207         0.04385
+        0.06207        -6.44758         0.05879
+       -0.11461         4.72937         0.05075
+       -0.28189        -5.30540         0.02418
+       -0.26693         3.40045         0.00139
+       -0.22230         3.23105        -0.00055
+       -0.18603         0.64944        -0.00157
+       -0.14637        -1.30498        -0.00206
+       -0.10033        -2.06939        -0.00199
+       -0.04705        -1.40579        -0.00106
+        0.00163         0.32470         0.00008
+------------------------------------------------
+bfgsmin iteration 14  convergence (f g p): 1 0 1
+
+function value: 7.65629  stepsize: 0.686701  
+
+used analytic gradient
+       -0.00233        -4.01674        -0.00177
+        0.05541         3.59409        -0.00268
+        0.10693         4.98945        -0.00269
+        0.14961         0.89269         0.00273
+        0.18984        -5.62038         0.00496
+        0.23919        -1.65027         0.00829
+        0.28737         4.10938         0.00931
+        0.31892        -1.10363         0.01880
+        0.33840         1.89994         0.02975
+        0.32500        -0.08734         0.05163
+        0.26196         1.75424         0.08332
+        0.12086        -5.14685         0.11526
+       -0.06386         3.50817         0.10028
+       -0.25771        -6.91246         0.05295
+       -0.26554         2.89865         0.00202
+       -0.22284         3.12119        -0.00248
+       -0.18760         0.35061        -0.00291
+       -0.14843        -1.71264        -0.00274
+       -0.10232        -2.46776        -0.00224
+       -0.04811        -1.62010        -0.00104
+        0.00171         0.34168        -0.00003
+------------------------------------------------
+bfgsmin iteration 15  convergence (f g p): 1 0 1
+
+function value: 7.60957  stepsize: 0.0878906  
+
+used analytic gradient
+       -0.00410        -4.35082         0.00146
+        0.05272         2.67307        -0.00072
+        0.10424         2.23962        -0.00136
+        0.15234         2.73648        -0.00123
+        0.19480        -5.11457         0.00013
+        0.24748         0.88470        -0.00259
+        0.29668         2.58823        -0.00490
+        0.33772         1.78585        -0.00824
+        0.36815         2.58724        -0.01376
+        0.37663         2.66756        -0.02323
+        0.34528         6.61338        -0.03712
+        0.23612         3.11979        -0.04848
+        0.03643        -0.09857        -0.04616
+       -0.20476       -11.64869        -0.01839
+       -0.26353         0.80826        -0.00333
+       -0.22532         2.63068        -0.00157
+       -0.19051        -0.19964         0.00073
+       -0.15117        -2.25741         0.00226
+       -0.10456        -2.92037         0.00266
+       -0.04915        -1.83321         0.00159
+        0.00169         0.33700        -0.00024
+------------------------------------------------
+bfgsmin iteration 16  convergence (f g p): 1 0 1
+
+function value: 7.41533  stepsize: 0.820359  
+
+used analytic gradient
+       -0.00263        -2.90226         0.00588
+        0.05200         1.90826        -0.00407
+        0.10288         1.64688        -0.00250
+        0.15111         1.99299        -0.00448
+        0.19493        -3.46470         0.00690
+        0.24489         0.20784        -0.00134
+        0.29178         2.02354        -0.00408
+        0.32948         1.08093        -0.00291
+        0.35439         1.76406        -0.00498
+        0.35340         1.19254        -0.00440
+        0.30816         3.02983        -0.00877
+        0.18764         0.33923        -0.00536
+       -0.00974        -0.50876        -0.00313
+       -0.22315        -7.70309         0.01212
+       -0.26686         1.18439        -0.00266
+       -0.22689         2.35832        -0.00475
+       -0.18978        -0.04625         0.00020
+       -0.14892        -1.80104         0.00379
+       -0.10190        -2.38287         0.00496
+       -0.04756        -1.50929         0.00313
+        0.00144         0.28696        -0.00059
+------------------------------------------------
+bfgsmin iteration 17  convergence (f g p): 1 0 1
+
+function value: 7.29262  stepsize: 0.355157  
+
+used analytic gradient
+        0.00324         3.44705        -0.00234
+        0.04793        -3.52336         0.00404
+        0.10038         2.55533        -0.00664
+        0.14663        -4.24460         0.00496
+        0.20183         5.76828        -0.00531
+        0.24355        -2.26044         0.00502
+        0.28770        -0.31176         0.00078
+        0.32657         1.82365        -0.00129
+        0.34942        -0.04595         0.00584
+        0.34900         2.48738         0.00813
+        0.29939        -0.00814         0.01998
+        0.18228         1.11516         0.02985
+       -0.01287        -2.06709         0.04066
+       -0.21103        -4.52312         0.04069
+       -0.26952         0.08239        -0.00000
+       -0.23165         1.44744        -0.00734
+       -0.18958         0.01731        -0.00034
+       -0.14513        -1.04350         0.00490
+       -0.09694        -1.39413         0.00669
+       -0.04443        -0.87916         0.00428
+        0.00085         0.16449        -0.00084
+------------------------------------------------
+bfgsmin iteration 18  convergence (f g p): 1 0 1
+
+function value: 7.11897  stepsize: 0.387671  
+
+used analytic gradient
+        0.00091        -0.05722        -0.00675
+        0.05196         4.11842        -0.00153
+        0.09373        -7.64764         0.00966
+        0.15159         5.48544        -0.00309
+        0.19652        -3.53802        -0.00377
+        0.24857         4.55574        -0.00324
+        0.28849        -1.03127         0.00538
+        0.32528        -2.00822         0.00324
+        0.35526         2.78635         0.00268
+        0.35713         0.01456         0.01093
+        0.31936         1.95859         0.02473
+        0.21213         0.29212         0.04011
+        0.02780        -2.69633         0.06367
+       -0.17034        -2.18041         0.05835
+       -0.26952        -2.42213         0.00717
+       -0.23899        -0.00494        -0.00873
+       -0.18992        -0.02677        -0.00040
+       -0.14023        -0.08385         0.00596
+       -0.09025        -0.08842         0.00810
+       -0.04016        -0.03002         0.00512
+        0.00001        -0.01653        -0.00095
+------------------------------------------------
+bfgsmin iteration 19  convergence (f g p): 1 0 1
+
+function value: 6.94514  stepsize: 2.53419  
+
+used analytic gradient
+       -0.00584        -4.81486         0.00521
+        0.05043         1.40122        -0.00186
+        0.10340         3.84808        -0.00582
+        0.14850         0.03207         0.00093
+        0.19275        -4.74812         0.00505
+        0.24533         0.67978        -0.00177
+        0.29386         4.29842        -0.00689
+        0.32853        -2.05982         0.00531
+        0.35794        -0.15275         0.00251
+        0.36806        -0.54721         0.00767
+        0.34409         3.40331         0.01044
+        0.25224        -2.17717         0.03490
+        0.09147        -0.65798         0.05086
+       -0.11199        -0.42017         0.04960
+       -0.26235        -5.73809         0.01820
+       -0.24772        -1.83103        -0.00395
+       -0.19032        -0.10645        -0.00004
+       -0.13427         1.04851         0.00328
+       -0.08216         1.45058         0.00436
+       -0.03504         0.96524         0.00264
+       -0.00093        -0.23171        -0.00037
+------------------------------------------------
+bfgsmin iteration 20  convergence (f g p): 1 0 1
+
+function value: 6.79893  stepsize: 3.10571  
+
+used analytic gradient
+       -0.00063         0.06865         0.00367
+        0.04856        -0.21388        -0.00202
+        0.09757        -1.62449         0.00213
+        0.14943         1.27893        -0.00328
+        0.19780         0.60135         0.00326
+        0.24356        -0.35140        -0.00292
+        0.28698        -3.89689         0.00316
+        0.33384         4.85705        -0.00596
+        0.36045        -2.83190         0.00614
+        0.37573         1.40645        -0.00044
+        0.35454        -3.13736         0.00850
+        0.28714         1.16118         0.01609
+        0.14233         0.28976         0.02990
+       -0.06239         0.36434         0.02828
+       -0.24416        -7.15621         0.03361
+       -0.25167        -2.95451         0.00613
+       -0.19036        -0.12628         0.00039
+       -0.13099         1.65590        -0.00276
+       -0.07780         2.26053        -0.00389
+       -0.03240         1.46787        -0.00268
+       -0.00131        -0.32307         0.00075
+------------------------------------------------
+bfgsmin iteration 21  convergence (f g p): 1 0 1
+
+function value: 6.63207  stepsize: 1.66919  
+
+used analytic gradient
+        0.00304         3.83407        -0.00349
+        0.04654        -4.54566         0.00488
+        0.09970         2.62065         0.00043
+        0.14615        -4.14731         0.00224
+        0.20106         6.35500        -0.00883
+        0.24064        -5.79584         0.00471
+        0.29014         2.72574        -0.00056
+        0.32788        -4.58532         0.00465
+        0.36658         5.47580         0.00490
+        0.37529        -4.48961         0.02583
+        0.36303        -1.79995         0.05399
+        0.30323        -0.28409         0.09944
+        0.17223         2.76649         0.15531
+       -0.03411        -2.37542         0.15925
+       -0.21054        -3.75942         0.10163
+       -0.24554        -2.71805         0.00844
+       -0.18997        -0.04245         0.00054
+       -0.13375         1.14465        -0.00052
+       -0.08169         1.53941        -0.00103
+       -0.03508         0.95134        -0.00116
+       -0.00056        -0.15653         0.00079
+------------------------------------------------
+bfgsmin iteration 22  convergence (f g p): 1 0 1
+
+function value: 6.50365  stepsize: 0.307514  
+
+used analytic gradient
+       -0.00045        -0.92799        -0.00085
+        0.05142         1.55785        -0.00010
+        0.10013         0.20377         0.00169
+        0.14838         1.43389        -0.00073
+        0.19223        -5.21443         0.00061
+        0.24534         2.60359        -0.00005
+        0.28958        -1.53872         0.00194
+        0.33253        -1.78661        -0.00143
+        0.37148        -1.69099        -0.00278
+        0.40112        -3.34120        -0.00660
+        0.41703        -1.99033        -0.01475
+        0.40267         1.88161        -0.03307
+        0.32755        15.32208        -0.05301
+        0.12514        -5.29606        -0.04861
+       -0.10891        -6.74691        -0.01969
+       -0.23711        -6.75195         0.00356
+       -0.18943         0.03667        -0.00000
+       -0.13427         1.04761        -0.00263
+       -0.08272         1.34837        -0.00347
+       -0.03624         0.71925        -0.00206
+        0.00023         0.00771         0.00023
+------------------------------------------------
+bfgsmin iteration 23  convergence (f g p): 1 0 1
+
+function value: 6.24282  stepsize: 3.3732  
+
+used analytic gradient
+       -0.00130        -1.56554         0.00672
+        0.05132         1.11771        -0.00200
+        0.10182         2.23793        -0.00484
+        0.14765        -0.22087        -0.00051
+        0.19285        -4.29563         0.01014
+        0.24529         1.53733        -0.00372
+        0.29151         0.95980        -0.00151
+        0.33110        -2.85378         0.00470
+        0.36870        -1.21267         0.00946
+        0.39453        -2.71530         0.01664
+        0.40228        -0.14777         0.02748
+        0.36960        -0.17686         0.05340
+        0.27453         7.58576         0.06945
+        0.07653        -5.04249         0.10220
+       -0.12860        -2.87700         0.08866
+       -0.23355        -4.62015         0.02548
+       -0.18943         0.02662         0.00072
+       -0.13690         0.55247        -0.00656
+       -0.08619         0.69234        -0.00904
+       -0.03830         0.32437        -0.00588
+        0.00046         0.06483         0.00128
+------------------------------------------------
+bfgsmin iteration 24  convergence (f g p): 1 0 1
+
+function value: 6.00496  stepsize: 0.477792  
+
+used analytic gradient
+        0.00542         4.65312        -0.00258
+        0.04933        -1.64034         0.00183
+        0.09698        -1.60799        -0.00195
+        0.14714        -2.83790         0.00491
+        0.20299         7.53060        -0.00501
+        0.24157        -5.59615         0.00676
+        0.29001        -0.91182        -0.00123
+        0.33579        -1.40600         0.00441
+        0.37817        -0.47388         0.00155
+        0.41117        -1.47962         0.00356
+        0.42976        -2.13551         0.00012
+        0.42300         7.78783        -0.01225
+        0.34399         3.23500        -0.01249
+        0.17874        -1.77263         0.00677
+       -0.03995        -2.70818         0.02349
+       -0.20807        -7.21570         0.02210
+       -0.18871        -0.09990         0.00047
+       -0.14346        -0.71601        -0.00396
+       -0.09523        -1.05590        -0.00527
+       -0.04418        -0.83308        -0.00309
+        0.00175         0.34233         0.00031
+------------------------------------------------
+bfgsmin iteration 25  convergence (f g p): 1 0 1
+
+function value: 5.76673  stepsize: 0.424707  
+
+used analytic gradient
+        0.00284         1.81972        -0.00254
+        0.05116         2.01699        -0.00259
+        0.09503        -6.23380         0.00757
+        0.15206         4.85833        -0.00485
+        0.19798        -2.17693         0.00256
+        0.24833         3.62200        -0.00230
+        0.28877        -6.55644         0.00956
+        0.34020         2.79814         0.00004
+        0.37971        -2.11144         0.00985
+        0.41472         1.21818         0.00998
+        0.42988         1.16392         0.01714
+        0.41074         2.16135         0.02504
+        0.33150        -2.53226         0.05107
+        0.18550        -0.15700         0.07025
+       -0.01646        -0.22112         0.06452
+       -0.18597        -5.34090         0.03367
+       -0.18824        -0.45654         0.00144
+       -0.14742        -1.50575        -0.00177
+       -0.10050        -2.10134        -0.00234
+       -0.04727        -1.45155        -0.00126
+        0.00205         0.40879        -0.00002
+------------------------------------------------
+bfgsmin iteration 26  convergence (f g p): 1 0 1
+
+function value: 5.57919  stepsize: 1.2178  
+
+used analytic gradient
+        0.00030         0.81315        -0.00377
+        0.04857        -2.58989         0.00240
+        0.10259         4.30190        -0.00212
+        0.14720        -4.04934         0.00333
+        0.20054         3.25221        -0.00146
+        0.24603        -3.51641         0.00401
+        0.29834         3.81782         0.00159
+        0.34025        -4.85936         0.01077
+        0.38956         3.56518        -0.00071
+        0.42471         0.22677         0.00312
+        0.44702         3.47990        -0.00747
+        0.43578        -1.71874         0.00030
+        0.38257         1.24036         0.00323
+        0.25576         2.45469         0.01943
+        0.04806        -1.81541         0.04420
+       -0.15230        -6.63652         0.05061
+       -0.18680        -1.24621         0.00431
+       -0.14919        -1.86962         0.00116
+       -0.10284        -2.57174         0.00181
+       -0.04853        -1.70620         0.00172
+        0.00203         0.40611        -0.00096
+------------------------------------------------
+bfgsmin iteration 27  convergence (f g p): 1 0 1
+
+function value: 5.32151  stepsize: 1.23225  
+
+used analytic gradient
+       -0.00348        -3.15946         0.00122
+        0.05097         2.16833        -0.00341
+        0.10048        -0.12317         0.00269
+        0.15053         0.71169        -0.00116
+        0.19908        -1.14912         0.00229
+        0.25004         0.43402         0.00152
+        0.29993        -0.49531         0.00808
+        0.35102         5.51214         0.00023
+        0.38885        -2.65542         0.01264
+        0.42782         6.44117         0.00176
+        0.43955        -4.68493         0.02212
+        0.43608         0.20113         0.03289
+        0.38580        -2.71638         0.06655
+        0.27518         0.43859         0.09068
+        0.09226        -0.06169         0.09995
+       -0.10169        -2.54934         0.07361
+       -0.18249        -2.87346         0.01005
+       -0.14803        -1.66191         0.00277
+       -0.10102        -2.20632         0.00387
+       -0.04681        -1.35790         0.00296
+        0.00107         0.21221        -0.00113
+------------------------------------------------
+bfgsmin iteration 28  convergence (f g p): 1 0 1
+
+function value: 5.1413  stepsize: 0.597099  
+
+used analytic gradient
+       -0.00225        -0.82707         0.00101
+        0.04756        -2.80033         0.00103
+        0.10317         4.41238        -0.00220
+        0.14937        -2.44571         0.00060
+        0.20137         1.00216         0.00047
+        0.25155        -2.19739         0.00018
+        0.30801         6.97457        -0.00371
+        0.35125        -2.55732        -0.00022
+        0.40149         9.17125        -0.00712
+        0.42959        -5.53508        -0.00228
+        0.46168         2.58999        -0.00932
+        0.46897        -4.21435        -0.01065
+        0.45235         4.90054        -0.02357
+        0.36586         3.10050        -0.02858
+        0.19222        -1.05583        -0.01939
+       -0.02808        -3.70302        -0.00167
+       -0.17244        -6.32629         0.00418
+       -0.14525        -1.19821         0.00188
+       -0.09715        -1.43577         0.00247
+       -0.04386        -0.76030         0.00152
+       -0.00006        -0.01863        -0.00024
+------------------------------------------------
+bfgsmin iteration 29  convergence (f g p): 1 0 1
+
+function value: 4.9937  stepsize: 2.17946  
+
+used analytic gradient
+       -0.00124        -0.43215        -0.00001
+        0.04859        -1.29829         0.00166
+        0.10097         1.55042        -0.00203
+        0.14997        -1.15813         0.00319
+        0.20184         1.16622        -0.00096
+        0.25173        -0.72380         0.00526
+        0.30430         3.22475         0.00040
+        0.35103         1.56389         0.00147
+        0.39437         3.01199         0.00304
+        0.42731        -1.25221         0.00889
+        0.45236        -1.16356         0.02116
+        0.45833        -1.26798         0.03706
+        0.42878         0.26780         0.06896
+        0.33729        -0.67125         0.10962
+        0.17283        -0.33471         0.13612
+       -0.02975        -0.74485         0.11489
+       -0.16826        -5.36274         0.03146
+       -0.14337        -0.87641         0.00609
+       -0.09468        -0.94998         0.00775
+       -0.04234        -0.45672         0.00535
+       -0.00029        -0.06993        -0.00153
+------------------------------------------------
+bfgsmin iteration 30  convergence (f g p): 1 0 1
+
+function value: 4.9757  stepsize: 0.2557  
+
+used analytic gradient
+       -0.00125        -1.10245         0.00086
+        0.05026         1.18300        -0.00034
+        0.09894        -2.42822         0.00039
+        0.15316         3.24745        -0.00115
+        0.20088        -3.17955         0.00062
+        0.25698         4.81326        -0.00247
+        0.30470         0.92442        -0.00189
+        0.35250         1.66812        -0.00323
+        0.39741         1.91989        -0.00324
+        0.43620        -2.11829        -0.00475
+        0.47351         1.03993        -0.00863
+        0.49539        -2.07961        -0.01632
+        0.49774         3.83376        -0.03011
+        0.44691         6.11178        -0.04336
+        0.30895         2.68674        -0.04587
+        0.08514        -1.88109        -0.03056
+       -0.13681       -11.91900        -0.00184
+       -0.13728        -0.29417        -0.00045
+       -0.08693         0.54123        -0.00082
+       -0.03698         0.60557        -0.00076
+       -0.00183        -0.39900         0.00041
+------------------------------------------------
+bfgsmin iteration 31  convergence (f g p): 1 0 1
+
+function value: 4.74007  stepsize: 2.01296  
+
+used analytic gradient
+       -0.00039        -0.27755         0.00144
+        0.04992         0.34424        -0.00080
+        0.09933        -1.44424         0.00433
+        0.15200         1.68174        -0.00473
+        0.20150        -1.10886         0.00342
+        0.25452         2.82255        -0.00819
+        0.30281         1.30446        -0.00429
+        0.34927         0.47118        -0.00482
+        0.39418         1.87683        -0.00678
+        0.43145        -2.07065         0.00310
+        0.46488         1.00031        -0.00241
+        0.47907        -2.18278         0.00642
+        0.46764         0.36474         0.00451
+        0.40355         1.09099         0.01248
+        0.26308         0.98767         0.03014
+        0.05457        -0.06190         0.04343
+       -0.13864        -8.36740         0.04358
+       -0.13773        -0.33346         0.00556
+       -0.08775         0.38477         0.00443
+       -0.03774         0.45320         0.00228
+       -0.00141        -0.31261         0.00011
+------------------------------------------------
+bfgsmin iteration 32  convergence (f g p): 1 0 1
+
+function value: 4.52865  stepsize: 0.429951  
+
+used analytic gradient
+        0.00106         1.19943        -0.00074
+        0.04912        -2.76071         0.00487
+        0.10365         5.12290        -0.00375
+        0.14727        -6.14816         0.00618
+        0.20492         7.52022        -0.01189
+        0.24633        -5.04269        -0.00280
+        0.29852         2.20086        -0.01136
+        0.34445         0.07698        -0.00735
+        0.38740        -4.11450         0.00092
+        0.43454         4.58369        -0.00314
+        0.46247        -4.96318         0.01349
+        0.48549         2.81820         0.01619
+        0.47215        -2.73398         0.03608
+        0.41603        -1.35444         0.05688
+        0.29321         1.39450         0.08884
+        0.09800         0.16651         0.10959
+       -0.09507        -5.31054         0.10251
+       -0.13217        -0.71739         0.01301
+       -0.08332         1.21002         0.00578
+       -0.03546         0.89019         0.00250
+       -0.00130        -0.30213         0.00074
+------------------------------------------------
+bfgsmin iteration 33  convergence (f g p): 1 0 1
+
+function value: 4.11409  stepsize: 0.657703  
+
+used analytic gradient
+        0.00031        -1.34372         0.00252
+        0.05399         3.93035        -0.00587
+        0.09990        -3.07964        -0.00065
+        0.15345         6.31359        -0.00719
+        0.19303        -5.74279        -0.00055
+        0.24353         1.45164         0.00222
+        0.28716        -4.99659         0.00683
+        0.33710        -3.01059         0.00411
+        0.38832         0.93546         0.00533
+        0.43140        -4.14058         0.00323
+        0.47597         2.83903         0.01022
+        0.50168        -1.22961         0.01309
+        0.50823         1.03823         0.03343
+        0.47291        -2.34812         0.06444
+        0.38205         2.41192         0.07531
+        0.20759        -2.24953         0.11161
+        0.00744        -0.96288         0.09248
+       -0.11916        -4.64238         0.02890
+       -0.07754         2.19808        -0.00308
+       -0.03296         1.34702        -0.00268
+       -0.00056        -0.16945         0.00112
+------------------------------------------------
+bfgsmin iteration 34  convergence (f g p): 1 0 1
+
+function value: 3.93492  stepsize: 0.583003  
+
+used analytic gradient
+        0.00284         3.03958        -0.00103
+        0.04812        -2.71622         0.00037
+        0.09926         1.49736        -0.00009
+        0.14626        -0.43109        -0.00032
+        0.19248        -4.29263         0.00375
+        0.24576         1.16017         0.00071
+        0.29399        -0.78655         0.00230
+        0.34121        -3.80186         0.00255
+        0.39365         3.28833        -0.00165
+        0.43464        -7.13741         0.00212
+        0.48619         6.38546        -0.00633
+        0.51477        -5.83356        -0.00453
+        0.54166         1.82499        -0.01373
+        0.53735        10.55949        -0.02471
+        0.45737        -5.15767        -0.02857
+        0.31920         3.73891        -0.03804
+        0.09992        -2.59094        -0.03129
+       -0.09026        -8.29990        -0.00235
+       -0.08062         1.09261        -0.00158
+       -0.03564         0.82404        -0.00072
+        0.00057         0.07222        -0.00028
+------------------------------------------------
+bfgsmin iteration 35  convergence (f g p): 1 0 1
+
+function value: 3.80188  stepsize: 1.1915  
+
+used analytic gradient
+        0.00181         2.05541        -0.00125
+        0.04849        -1.89628         0.00030
+        0.09917         1.38904        -0.00174
+        0.14595        -2.20622         0.00051
+        0.19623        -0.73899        -0.00139
+        0.24647        -0.53583         0.00034
+        0.29629         0.19805         0.00009
+        0.34376        -1.54737         0.00132
+        0.39199        -0.20173         0.00279
+        0.43676        -1.91790         0.00250
+        0.47986         1.14865         0.00623
+        0.51023        -2.31606         0.01150
+        0.52793         0.44128         0.02091
+        0.51265         5.34635         0.03336
+        0.42880        -3.62154         0.05537
+        0.28116         1.93650         0.06869
+        0.06862        -2.57774         0.06387
+       -0.09261        -5.17205         0.02062
+       -0.08220         0.84185        -0.00073
+       -0.03635         0.69285        -0.00123
+        0.00029         0.02113         0.00056
+------------------------------------------------
+bfgsmin iteration 36  convergence (f g p): 1 0 1
+
+function value: 3.74259  stepsize: 0.740651  
+
+used analytic gradient
+        0.00056         0.92800        -0.00023
+        0.04880        -0.39921        -0.00142
+        0.09743        -0.67589         0.00297
+        0.14646        -0.44778         0.00013
+        0.19484        -2.45603         0.00438
+        0.24682         0.32147         0.00039
+        0.29638        -0.37433         0.00181
+        0.34507        -1.38342         0.00121
+        0.39478         1.04005         0.00007
+        0.43926        -3.03502         0.00106
+        0.48609         1.71091        -0.00253
+        0.52174        -1.95768        -0.00459
+        0.54884         2.43981        -0.01391
+        0.54601         5.62285        -0.02158
+        0.48417         0.55973        -0.02722
+        0.34985         2.74674        -0.03287
+        0.13249        -2.47424        -0.02219
+       -0.07199        -8.69540         0.00639
+       -0.08293         0.14815        -0.00101
+       -0.03758         0.44691        -0.00095
+        0.00086         0.14023        -0.00036
+------------------------------------------------
+bfgsmin iteration 37  convergence (f g p): 1 0 1
+
+function value: 3.59675  stepsize: 1.01761  
+
+used analytic gradient
+        0.00033         1.31232        -0.00302
+        0.04738        -2.91301         0.00461
+        0.10040         2.81763        -0.00508
+        0.14659        -3.25776         0.00638
+        0.19923         1.70235        -0.00230
+        0.24721        -1.75515         0.00432
+        0.29819         0.79090        -0.00002
+        0.34628        -0.93398         0.00346
+        0.39485         0.20803         0.00202
+        0.44032        -1.02082         0.00440
+        0.48356         0.62864         0.00310
+        0.51715         0.05614         0.00586
+        0.53494        -0.33892         0.01057
+        0.52443         2.49842         0.01344
+        0.45695        -0.21019         0.03054
+        0.31699         0.47060         0.04129
+        0.11030        -2.13691         0.04741
+       -0.06561        -4.51163         0.03123
+       -0.08394        -0.25260        -0.00044
+       -0.03853         0.26389        -0.00208
+        0.00049         0.07220         0.00013
+------------------------------------------------
+bfgsmin iteration 38  convergence (f g p): 1 0 1
+
+function value: 3.46447  stepsize: 0.693985  
+
+used analytic gradient
+       -0.00269        -2.93985         0.00184
+        0.05198         4.94004        -0.00230
+        0.09532        -6.63311         0.00721
+        0.15297         6.07971         0.00046
+        0.19693        -4.86341         0.00864
+        0.25153         3.49324         0.00372
+        0.29817        -2.33545         0.00916
+        0.34974         1.72147         0.00427
+        0.39687        -0.91016         0.01195
+        0.44472         1.29504         0.00641
+        0.48666        -0.39147         0.01219
+        0.52301         0.32948         0.01078
+        0.54551         1.95649         0.02165
+        0.53786        -1.15706         0.04029
+        0.48749         3.14657         0.06914
+        0.35827        -0.16542         0.11924
+        0.15771        -1.97732         0.15013
+       -0.03438        -4.33943         0.12038
+       -0.08438        -1.57633         0.00534
+       -0.04061        -0.15352        -0.00659
+        0.00062         0.10646        -0.00018
+------------------------------------------------
+bfgsmin iteration 39  convergence (f g p): 1 0 1
+
+function value: 3.17289  stepsize: 0.51348  
+
+used analytic gradient
+       -0.00085        -0.55285         0.00176
+        0.04969        -0.99136        -0.00018
+        0.10254         1.30388         0.00030
+        0.15342         0.19336        -0.00119
+        0.20557         2.13395        -0.00181
+        0.25525         0.10909        -0.00047
+        0.30734         3.69435        -0.00277
+        0.35401        -2.45205         0.00017
+        0.40882         6.87229        -0.00522
+        0.45113        -1.95140        -0.00198
+        0.49885         4.87700        -0.00521
+        0.53378        -2.54942        -0.00443
+        0.56715         2.59602        -0.01020
+        0.57815         0.20660        -0.01231
+        0.55663         0.90471        -0.02537
+        0.47751         4.43110        -0.03326
+        0.30784        -0.82222        -0.02917
+        0.08600        -2.62801        -0.01149
+       -0.07904        -8.92593         0.00739
+       -0.04720        -1.52261         0.00187
+        0.00044         0.08605        -0.00025
+------------------------------------------------
+bfgsmin iteration 40  convergence (f g p): 1 0 1
+
+function value: 3.02629  stepsize: 1.49845  
+
+used analytic gradient
+        0.00091         0.92588        -0.00090
+        0.04951        -1.98902         0.00286
+        0.10284         2.15468        -0.00119
+        0.15223        -0.40464         0.00222
+        0.20376         0.97320         0.00127
+        0.25478         1.47491         0.00071
+        0.30457         1.00348         0.00201
+        0.35418         0.93636         0.00148
+        0.40360         2.28759         0.00156
+        0.44915         0.24517         0.00294
+        0.49364         2.22357         0.00207
+        0.52935        -0.77884         0.00611
+        0.55695        -0.55658         0.01065
+        0.56584         2.60055         0.01483
+        0.53125        -2.92529         0.03642
+        0.44425         0.96023         0.05399
+        0.27867        -0.82084         0.07230
+        0.07451        -0.49244         0.05843
+       -0.07166        -6.39978         0.01270
+       -0.04532        -1.21660        -0.00134
+        0.00019         0.03401        -0.00013
+------------------------------------------------
+bfgsmin iteration 41  convergence (f g p): 1 0 1
+
+function value: 2.95985  stepsize: 0.457506  
+
+used analytic gradient
+        0.00001        -0.93732         0.00197
+        0.05237         1.71490        -0.00283
+        0.10165        -1.07599         0.00248
+        0.15445         1.79921        -0.00441
+        0.20503         1.08421        -0.00074
+        0.25549         0.87292        -0.00376
+        0.30658         2.16220        -0.00223
+        0.35566         0.99421        -0.00477
+        0.40516         2.09347        -0.00269
+        0.45208         1.74382        -0.00629
+        0.49571         0.67623        -0.00248
+        0.53547         0.17296        -0.00513
+        0.56760         1.38862        -0.00721
+        0.58067        -1.82298        -0.00851
+        0.56767         1.64060        -0.01829
+        0.49824         1.23723        -0.02348
+        0.35097         1.86798        -0.02400
+        0.13294        -1.14405        -0.00815
+       -0.05896        -9.74720         0.01780
+       -0.04667        -1.66341         0.00359
+        0.00006         0.00964        -0.00017
+------------------------------------------------
+bfgsmin iteration 42  convergence (f g p): 1 0 1
+
+function value: 2.79608  stepsize: 0.612391  
+
+used analytic gradient
+        0.00199         1.78135        -0.00233
+        0.04954        -2.90645         0.00430
+        0.10413         4.32872        -0.00580
+        0.15003        -3.33958         0.00486
+        0.20430         3.62049        -0.00543
+        0.25173        -1.73066         0.00226
+        0.30435         3.32930        -0.00517
+        0.35089        -1.83369         0.00161
+        0.40246         3.80565        -0.00510
+        0.44579        -2.48173         0.00288
+        0.49322         2.75760        -0.00234
+        0.53034        -1.01041         0.00381
+        0.56039        -0.13811         0.00629
+        0.57216         0.17525         0.01192
+        0.54938        -1.62740         0.02590
+        0.47476        -0.37717         0.04094
+        0.32697         0.00266         0.05416
+        0.12479        -0.19824         0.05088
+       -0.04115        -5.29730         0.03078
+       -0.04307        -1.29237         0.00294
+       -0.00011        -0.03137        -0.00011
+------------------------------------------------
+bfgsmin iteration 43  convergence (f g p): 1 0 1
+
+function value: 2.61633  stepsize: 1.05544  
+
+used analytic gradient
+       -0.00034        -1.80918        -0.00099
+        0.05384         4.66848        -0.00286
+        0.09833        -5.15901         0.00002
+        0.15489         6.05105        -0.00760
+        0.19887        -4.68652         0.00084
+        0.25399         4.80028        -0.01252
+        0.29918        -3.42643        -0.00233
+        0.35251         3.90679        -0.01159
+        0.39736        -3.10992        -0.00452
+        0.44867         3.36452        -0.00577
+        0.49089        -2.21543        -0.00234
+        0.53415         1.15603         0.00091
+        0.56669        -0.28282         0.01155
+        0.58408        -1.24687         0.02077
+        0.57528         0.45045         0.04716
+        0.51570         1.03135         0.07414
+        0.38112         0.92477         0.10727
+        0.17567        -1.61367         0.12067
+       -0.01037        -5.17352         0.10233
+       -0.04014        -1.61271         0.01561
+       -0.00022        -0.06324        -0.00016
+------------------------------------------------
+bfgsmin iteration 44  convergence (f g p): 1 0 1
+
+function value: 2.24975  stepsize: 1.15129  
+
+used analytic gradient
+       -0.00133        -1.45441         0.00198
+        0.05098         2.17057        -0.00448
+        0.09835        -0.95643         0.00187
+        0.14729        -1.93468        -0.00091
+        0.19971         4.20744        -0.00495
+        0.24147        -7.09614         0.00463
+        0.29685         3.87289        -0.00543
+        0.34092        -4.90549         0.00173
+        0.39284        -0.67911        -0.00011
+        0.44290         0.35038        -0.00412
+        0.48855        -2.58077         0.00184
+        0.53505        -1.57643        -0.00062
+        0.57823         2.35175        -0.00389
+        0.60486        -4.59037         0.00407
+        0.62244         6.06681        -0.00828
+        0.58983         3.15398        -0.00354
+        0.48839         3.57312         0.00294
+        0.29634        -5.31849         0.02753
+        0.09196        -2.19479         0.03215
+       -0.02453        -4.21387         0.01190
+       -0.00038        -0.20549         0.00024
+------------------------------------------------
+bfgsmin iteration 45  convergence (f g p): 1 0 1
+
+function value: 2.02508  stepsize: 1.02581  
+
+used analytic gradient
+        0.00065         1.92187        -0.00173
+        0.04650        -3.84373         0.00091
+        0.10022         3.07125        -0.00345
+        0.14637        -1.61790        -0.00096
+        0.19476        -2.21834        -0.00235
+        0.24610         1.62459        -0.00375
+        0.29141        -4.04191        -0.00244
+        0.34265        -0.99620        -0.00303
+        0.39272         0.15895        -0.00391
+        0.43878        -4.39080        -0.00027
+        0.49039         1.11168        -0.00146
+        0.53443        -1.38155         0.00300
+        0.57434        -2.73820         0.00864
+        0.60893         3.89418         0.01462
+        0.61417        -1.46042         0.03090
+        0.58629         2.48231         0.05038
+        0.49133        -1.10263         0.07899
+        0.32387        -1.78111         0.09783
+        0.12411        -0.48834         0.07954
+       -0.01263        -4.47346         0.01894
+       -0.00014        -0.30800         0.00027
+------------------------------------------------
+bfgsmin iteration 46  convergence (f g p): 1 0 1
+
+function value: 1.96678  stepsize: 0.308731  
+
+used analytic gradient
+       -0.00109         0.16353         0.00076
+        0.04741        -0.86351        -0.00037
+        0.09677        -0.36168         0.00167
+        0.14542        -0.25677         0.00016
+        0.19241        -2.66869         0.00220
+        0.24235        -0.19589         0.00184
+        0.28898        -3.74271         0.00315
+        0.33961        -1.46600         0.00231
+        0.38881        -2.38173         0.00307
+        0.43851        -2.52539         0.00120
+        0.48893        -1.40748         0.00162
+        0.53744        -1.27372        -0.00066
+        0.58298        -1.31475        -0.00333
+        0.62355         2.46885        -0.00783
+        0.64507         1.79501        -0.01639
+        0.63667         4.10533        -0.02613
+        0.57032         3.04951        -0.03483
+        0.42170         0.10881        -0.03422
+        0.20365        -1.67121        -0.02467
+        0.00631        -8.99724         0.00048
+        0.00013        -0.60894         0.00031
+------------------------------------------------
+bfgsmin iteration 47  convergence (f g p): 1 0 1
+
+function value: 1.82628  stepsize: 2.70162  
+
+used analytic gradient
+       -0.00032         0.92364        -0.00204
+        0.04704        -2.20471         0.00280
+        0.09844         1.39269        -0.00446
+        0.14557        -1.64313         0.00285
+        0.19461        -1.28472        -0.00160
+        0.24419        -0.50297         0.00282
+        0.29212        -2.29376         0.00115
+        0.34192        -1.65520         0.00436
+        0.39188        -0.75786         0.00020
+        0.43971        -3.20235         0.00559
+        0.49055        -0.03206         0.00110
+        0.53677        -1.24805         0.00497
+        0.57965        -1.20704         0.00556
+        0.61571         2.64735         0.00415
+        0.62868        -0.22331         0.01164
+        0.61055         1.08089         0.02469
+        0.53549        -0.52041         0.05719
+        0.38748         0.51139         0.09471
+        0.17898        -1.07463         0.09447
+        0.00679        -6.03139         0.04681
+        0.00044        -0.55768         0.00233
+------------------------------------------------
+bfgsmin iteration 48  convergence (f g p): 1 0 1
+
+function value: 1.66493  stepsize: 0.334132  
+
+used analytic gradient
+       -0.00236        -1.82523         0.00065
+        0.04984         3.19233        -0.00112
+        0.09398        -5.29754         0.00394
+        0.14843         3.61836        -0.00281
+        0.19301        -5.12741         0.00794
+        0.24701         2.47848        -0.00312
+        0.29327        -4.01346         0.00849
+        0.34629         2.13291        -0.00050
+        0.39208        -4.51277         0.00648
+        0.44530         1.80014         0.00227
+        0.49164        -3.14039         0.00405
+        0.54175         1.00787         0.00068
+        0.58521         0.60286         0.00142
+        0.61986        -0.09046        -0.00260
+        0.64032        -0.57143         0.00285
+        0.63523        -3.22361         0.00859
+        0.59268         0.17332         0.01755
+        0.48218         6.65993         0.02245
+        0.27345        -1.13512         0.03759
+        0.05359        -8.63737         0.03427
+        0.00277        -1.59232         0.00379
+------------------------------------------------
+bfgsmin iteration 49  convergence (f g p): 1 0 1
+
+function value: 1.47577  stepsize: 0.469317  
+
+used analytic gradient
+       -0.00171        -0.85954         0.00169
+        0.04873         0.23919        -0.00079
+        0.09792         0.18377         0.00313
+        0.14561        -3.91933         0.00479
+        0.20095         5.15204        -0.00035
+        0.24388        -7.16609         0.00975
+        0.30176         5.90020        -0.00078
+        0.34578        -4.33753         0.00799
+        0.39856         1.21642         0.00522
+        0.44757        -0.13885         0.00433
+        0.49570        -0.29921         0.00521
+        0.54242        -0.49886         0.00257
+        0.58663         2.75846        -0.00238
+        0.61726        -4.23145         0.00553
+        0.64317        -0.20792         0.00830
+        0.64383        -3.39838         0.02558
+        0.61022         2.47268         0.03259
+        0.50463         3.52631         0.04020
+        0.31104         0.43370         0.06568
+        0.08786        -7.25654         0.06451
+        0.00656        -2.48856         0.00998
+------------------------------------------------
+bfgsmin iteration 50  convergence (f g p): 1 0 1
+
+function value: 1.2336  stepsize: 0.841511  
+
+used analytic gradient
+       -0.00002         0.81156        -0.00041
+        0.04794        -2.47514         0.00380
+        0.10106         1.71922        -0.00079
+        0.15041        -0.25282         0.00268
+        0.20060        -1.01643         0.00238
+        0.25363         3.02329        -0.00065
+        0.30098        -1.98298         0.00346
+        0.35377         1.88930         0.00048
+        0.40378         1.53057         0.00028
+        0.45189         0.02368         0.00205
+        0.50091         2.15787        -0.00129
+        0.54499         0.91489        -0.00109
+        0.58425        -2.77153         0.00192
+        0.62279        -1.24847         0.00163
+        0.65147        -4.43949         0.00685
+        0.66941         3.57880        -0.00182
+        0.64282         4.41432        -0.00781
+        0.54483        -2.02397        -0.00469
+        0.37672         2.48643         0.00192
+        0.15237        -4.59893         0.02744
+        0.01655        -4.88103         0.01176
+------------------------------------------------
+bfgsmin iteration 51  convergence (f g p): 1 0 1
+
+function value: 1.09687  stepsize: 3.43706  
+
+used analytic gradient
+       -0.00043        -1.03597         0.00309
+        0.05174         1.80754        -0.00286
+        0.10027        -1.66240         0.00571
+        0.15309         1.80237        -0.00129
+        0.20298         0.56505         0.00153
+        0.25298         0.02512         0.00351
+        0.30444         1.56518        -0.00023
+        0.35426         0.87918         0.00198
+        0.40406         0.79307         0.00218
+        0.45394         2.49771        -0.00206
+        0.49962         0.47701         0.00161
+        0.54390        -0.42186         0.00220
+        0.58618        -1.09759         0.00338
+        0.62442        -3.11515         0.01072
+        0.65832         1.97843         0.00423
+        0.66759         2.23758         0.00970
+        0.63501         0.64728         0.01880
+        0.54014        -2.68923         0.03633
+        0.37865        -0.70654         0.05069
+        0.17981         0.85230         0.05453
+        0.02831        -4.89984         0.02402
+------------------------------------------------
+bfgsmin iteration 52  convergence (f g p): 1 0 1
+
+function value: 0.985596  stepsize: 1.74591  
+
+used analytic gradient
+        0.00267         2.59792        -0.00060
+        0.04888        -4.58169         0.00427
+        0.10598         5.75976        -0.00379
+        0.15180        -2.40009         0.00159
+        0.20451         1.21391        -0.00218
+        0.25650         3.05780        -0.00212
+        0.30421        -0.87498        -0.00113
+        0.35624         2.09802        -0.00328
+        0.40624         3.03838        -0.00557
+        0.45188        -1.10237        -0.00361
+        0.50123         2.03604        -0.00343
+        0.54610        -0.21695         0.00366
+        0.58956        -2.89428         0.01151
+        0.63514         4.28614         0.00699
+        0.66254        -1.97662         0.00484
+        0.67730         2.26832         0.00070
+        0.65381        -0.64605         0.01814
+        0.57647        -1.51083         0.04318
+        0.42934        -1.49962         0.05781
+        0.23434         2.63836         0.05439
+        0.05233        -5.70348         0.04833
+------------------------------------------------
+bfgsmin iteration 53  convergence (f g p): 1 0 1
+
+function value: 0.788855  stepsize: 1.86942  
+
+used analytic gradient
+        0.00206         0.39702         0.00055
+        0.05315         1.46549        -0.00239
+        0.10219        -0.42477         0.00221
+        0.15339         1.59995        -0.00343
+        0.20233        -0.77281         0.00180
+        0.25438         2.23828        -0.00483
+        0.30308         0.15766        -0.00110
+        0.35296         1.47091        -0.00466
+        0.40067         0.17993        -0.00281
+        0.44828        -1.11325        -0.00037
+        0.49779        -1.42142         0.00308
+        0.54977         0.22241         0.00431
+        0.60106         4.31359        -0.00059
+        0.64212         4.70269        -0.00020
+        0.66738        -0.42550         0.00940
+        0.67800        -5.70161         0.02708
+        0.67194        -1.37014         0.03668
+        0.61965         3.01636         0.05767
+        0.48715        -0.45247         0.09436
+        0.28872        -0.62200         0.09495
+        0.10066        -2.81347         0.06705
+------------------------------------------------
+bfgsmin iteration 54  convergence (f g p): 1 0 1
+
+function value: 0.648778  stepsize: 0.568461  
+
+used analytic gradient
+        0.00262         1.80337        -0.00139
+        0.05076        -2.05052         0.00049
+        0.10440         4.14478        -0.00263
+        0.14996        -3.45319         0.00082
+        0.20413         4.35393        -0.00233
+        0.24955        -2.89753         0.00001
+        0.30198         2.85200        -0.00168
+        0.34830        -1.63884        -0.00012
+        0.39787        -0.61797        -0.00027
+        0.44791        -1.59083         0.00082
+        0.50088         0.08233        -0.00038
+        0.55408         3.56937        -0.00260
+        0.60047         2.06974        -0.00343
+        0.64192         1.00792        -0.00439
+        0.67677        -1.86128        -0.00265
+        0.70508         1.47147        -0.00557
+        0.70862        -1.52321        -0.00996
+        0.67732         0.97438        -0.02012
+        0.58151         7.16637        -0.03094
+        0.38367        -3.75871        -0.02796
+        0.16772        -4.07107        -0.01192
+------------------------------------------------
+bfgsmin iteration 55  convergence (f g p): 1 0 1
+
+function value: 0.579233  stepsize: 1.3916  
+
+used analytic gradient
+        0.00122         0.48523         0.00001
+        0.05124         0.04672        -0.00143
+        0.10177         0.97224        -0.00009
+        0.15078        -0.64983        -0.00113
+        0.20180         1.67140        -0.00242
+        0.24957        -1.27528        -0.00017
+        0.30030         1.19985        -0.00323
+        0.34819        -0.97588        -0.00108
+        0.39760        -1.16432        -0.00002
+        0.44873        -0.51338        -0.00101
+        0.50050         0.41692        -0.00032
+        0.55148         2.46819        -0.00163
+        0.59704         1.42775         0.00034
+        0.63753        -0.90636         0.00595
+        0.67413        -0.50154         0.00637
+        0.69952         1.12854         0.01131
+        0.69866        -0.64458         0.02493
+        0.65719        -0.14683         0.04336
+        0.55058         3.66861         0.05865
+        0.35571        -3.26733         0.07228
+        0.15579        -1.76156         0.05134
+------------------------------------------------
+bfgsmin iteration 56  convergence (f g p): 1 0 1
+
+function value: 0.528891  stepsize: 0.598822  
+
+used analytic gradient
+        0.00124         1.06791        -0.00114
+        0.04981        -1.35475        -0.00042
+        0.10169         1.90618        -0.00134
+        0.14965        -0.77693        -0.00140
+        0.19938        -0.23872         0.00044
+        0.24940         0.81234        -0.00249
+        0.29707        -1.52703         0.00133
+        0.34711        -0.75377        -0.00003
+        0.39758        -0.35078         0.00079
+        0.44772        -1.38106         0.00106
+        0.50017         1.14526        -0.00208
+        0.54985         0.82473        -0.00264
+        0.59738         0.04751        -0.00267
+        0.64348         2.31578        -0.00383
+        0.68050        -1.12275        -0.00190
+        0.71083        -0.34677        -0.00344
+        0.72359         0.84317        -0.00846
+        0.70055         4.04183        -0.01727
+        0.60923         3.67045        -0.02331
+        0.42799        -1.96370        -0.01613
+        0.20713        -4.26991        -0.00280
+------------------------------------------------
+bfgsmin iteration 57  convergence (f g p): 1 0 1
+
+function value: 0.457551  stepsize: 0.881098  
+
+used analytic gradient
+        0.00010         0.32159        -0.00064
+        0.04939        -0.78770         0.00035
+        0.10035         1.29379        -0.00227
+        0.14825        -1.81836         0.00175
+        0.19982         1.75593        -0.00360
+        0.24690        -2.38165         0.00286
+        0.29840         0.80457        -0.00214
+        0.34708        -1.62784         0.00210
+        0.39837         0.02706        -0.00030
+        0.44878         0.18624        -0.00103
+        0.49810        -0.29801        -0.00029
+        0.54721         0.07936        -0.00061
+        0.59472        -0.02802         0.00040
+        0.63965         0.33915         0.00165
+        0.67860        -0.09969         0.00357
+        0.70739         0.41632         0.00650
+        0.71514         1.07612         0.01133
+        0.68328         1.63975         0.02009
+        0.58592         0.04937         0.03464
+        0.41187        -1.13856         0.04486
+        0.20433        -1.79889         0.03702
+------------------------------------------------
+bfgsmin iteration 58  convergence (f g p): 1 0 1
+
+function value: 0.413963  stepsize: 0.818213  
+
+used analytic gradient
+       -0.00055        -0.33431        -0.00023
+        0.04974         0.72900        -0.00319
+        0.09808        -1.81832         0.00126
+        0.15000         2.27845        -0.00448
+        0.19622        -3.67847         0.00291
+        0.24976         2.77059        -0.00214
+        0.29626        -3.31201         0.00474
+        0.34918         1.44073         0.00025
+        0.39808        -0.69586         0.00207
+        0.44775        -0.60415        -0.00107
+        0.49780         0.06379        -0.00111
+        0.54660        -0.57537        -0.00124
+        0.59512        -0.03985        -0.00189
+        0.64131         0.39631         0.00088
+        0.68217         0.16273         0.00285
+        0.71388         0.77608         0.00538
+        0.72647         1.17672         0.00908
+        0.70337         1.33604         0.01992
+        0.62056         1.49791         0.04198
+        0.45673        -0.35387         0.06887
+        0.24135        -3.08528         0.07380
+------------------------------------------------
+bfgsmin iteration 59  convergence (f g p): 1 0 1
+
+function value: 0.31226  stepsize: 0.967999  
+
+used analytic gradient
+       -0.00078         0.75551        -0.00127
+        0.04655        -2.86586         0.00238
+        0.09934         2.51020        -0.00272
+        0.14551        -3.86031         0.00281
+        0.19913         1.86329         0.00023
+        0.24763        -2.41740         0.00229
+        0.30099         2.17595         0.00042
+        0.34942        -1.03400         0.00135
+        0.40015         1.70805        -0.00192
+        0.44668        -2.05806         0.00319
+        0.49669        -0.12108        -0.00036
+        0.54536        -0.61164         0.00111
+        0.59323        -1.76779         0.00288
+        0.64218         0.88276         0.00082
+        0.68502         0.48596         0.00267
+        0.71926         1.25423         0.00488
+        0.73555        -0.17886         0.01505
+        0.72330        -0.60577         0.03277
+        0.66253         1.28843         0.05515
+        0.52560         1.70336         0.08160
+        0.31515        -3.23412         0.09360
+------------------------------------------------
+bfgsmin iteration 60  convergence (f g p): 1 0 1
+
+function value: 0.243113  stepsize: 0.473312  
+
+used analytic gradient
+       -0.00205        -1.21174         0.00072
+        0.04894         1.10347        -0.00023
+        0.09662        -2.27580         0.00157
+        0.14832        -0.07229         0.00006
+        0.19935         0.05454         0.00103
+        0.24992        -0.38823        -0.00019
+        0.30142         1.14335         0.00001
+        0.35078         0.91885        -0.00086
+        0.39823        -2.02547         0.00082
+        0.44987         2.04096        -0.00027
+        0.49633        -2.20147         0.00126
+        0.54647        -0.50754         0.00058
+        0.59611         0.32125        -0.00019
+        0.64300        -0.51049        -0.00067
+        0.68769         0.85143        -0.00232
+        0.72414        -0.96831        -0.00315
+        0.75060        -0.69511        -0.00556
+        0.75606         1.48057        -0.01055
+        0.71768         2.71311        -0.01769
+        0.60720         3.12324        -0.02219
+        0.40875        -4.63313        -0.01346
+------------------------------------------------
+bfgsmin iteration 61  convergence (f g p): 1 0 1
+
+function value: 0.20315  stepsize: 1.52488  
+
+used analytic gradient
+       -0.00133        -0.54472         0.00027
+        0.04870        -0.04351        -0.00026
+        0.09819        -0.64345         0.00042
+        0.14838        -1.05402         0.00174
+        0.20039         1.14333        -0.00072
+        0.24973        -0.99183         0.00288
+        0.30142         1.56996        -0.00060
+        0.34991        -0.27810         0.00171
+        0.39905        -0.75630         0.00163
+        0.44960         0.94140        -0.00059
+        0.49759        -1.06656         0.00169
+        0.54705        -0.36009         0.00083
+        0.59592         0.16867         0.00037
+        0.64233        -0.17825         0.00143
+        0.68538         0.08822         0.00266
+        0.72100        -0.90830         0.00772
+        0.74504        -0.58278         0.01591
+        0.74551         0.97336         0.03078
+        0.69999         0.79923         0.05892
+        0.58502         0.29621         0.09430
+        0.39529        -1.59020         0.11270
+------------------------------------------------
+bfgsmin iteration 62  convergence (f g p): 1 0 1
+
+function value: 0.173298  stepsize: 0.416513  
+
+used analytic gradient
+       -0.00105        -0.22332         0.00008
+        0.04845        -0.57811         0.00111
+        0.09861        -0.81192         0.00015
+        0.15012         0.80404         0.00041
+        0.19967        -1.42545         0.00033
+        0.25261         2.42814        -0.00109
+        0.30082        -0.86811        -0.00008
+        0.35162         1.02490        -0.00016
+        0.40068         0.42505        -0.00064
+        0.44901        -0.97223         0.00111
+        0.49929         0.52538        -0.00042
+        0.54788        -0.35397         0.00028
+        0.59630        -0.35890        -0.00003
+        0.64376         0.02701        -0.00094
+        0.68804        -0.91249        -0.00054
+        0.72871        -0.74794        -0.00206
+        0.76095        -0.56412        -0.00471
+        0.77629         0.09203        -0.00960
+        0.75891         2.78593        -0.01674
+        0.67931         4.75173        -0.02203
+        0.50799        -4.78149        -0.01522
+------------------------------------------------
+bfgsmin iteration 63  convergence (f g p): 1 0 1
+
+function value: 0.131239  stepsize: 0.701367  
+
+used analytic gradient
+       -0.00097        -0.60266         0.00089
+        0.04956         0.44244        -0.00058
+        0.09877        -1.26891         0.00198
+        0.15052         1.01918        -0.00127
+        0.20000        -0.82019         0.00153
+        0.25152         1.22736        -0.00139
+        0.30075        -0.44138         0.00109
+        0.35146         1.14619        -0.00128
+        0.40004        -0.59054         0.00115
+        0.45012         0.56058        -0.00053
+        0.49886        -0.45245         0.00090
+        0.54816         0.10812         0.00006
+        0.59627        -0.12741         0.00048
+        0.64282        -0.67333         0.00153
+        0.68749        -0.26193         0.00156
+        0.72665        -0.67752         0.00362
+        0.75624        -0.48550         0.00641
+        0.76668        -0.39006         0.01244
+        0.74217         0.71600         0.02130
+        0.65728         1.13793         0.03459
+        0.49278        -1.49535         0.04627
+------------------------------------------------
+bfgsmin iteration 64  convergence (f g p): 1 0 1
+
+function value: 0.113519  stepsize: 1.08423  
+
+used analytic gradient
+       -0.00008         0.34431         0.00141
+        0.04899        -1.28194         0.00263
+        0.10074         1.45197         0.00070
+        0.14925        -1.65936         0.00220
+        0.20152         1.77794        -0.00196
+        0.25013        -1.21475         0.00059
+        0.30184         1.72733        -0.00294
+        0.35018        -1.03959        -0.00003
+        0.40119         1.29150        -0.00089
+        0.44959        -0.79251         0.00089
+        0.49976         0.63795         0.00063
+        0.54822        -0.38905         0.00110
+        0.59675        -0.27771         0.00173
+        0.64435         0.03187         0.00321
+        0.68905        -0.77157         0.00483
+        0.73027        -0.30264         0.00730
+        0.76265        -0.66582         0.01262
+        0.77912         0.27018         0.02041
+        0.76347         1.04609         0.03451
+        0.69187         2.10541         0.06028
+        0.53904        -2.26629         0.10094
+------------------------------------------------
+bfgsmin iteration 65  convergence (f g p): 1 0 1
+
+function value: 0.0611149  stepsize: 0.729938  
+
+used analytic gradient
+        0.00133         0.42272        -0.00013
+        0.05162         0.51012         0.00039
+        0.10144         0.22350        -0.00033
+        0.15145         1.04764        -0.00045
+        0.19957        -1.30121         0.00103
+        0.25072         1.33787        -0.00140
+        0.29890        -1.44484         0.00096
+        0.35015         0.46339        -0.00044
+        0.40030         0.05289        -0.00049
+        0.45048         0.20528        -0.00001
+        0.50039         0.47081        -0.00042
+        0.54932        -0.23356         0.00104
+        0.59848        -0.27006         0.00149
+        0.64756         0.61439         0.00082
+        0.69388        -0.17544         0.00325
+        0.73757        -0.05419         0.00494
+        0.77527         0.57127         0.00792
+        0.79953         0.95810         0.01381
+        0.79798         0.58167         0.02642
+        0.75215        -0.12841         0.04651
+        0.63999        -0.69172         0.06967
+------------------------------------------------
+bfgsmin iteration 66  convergence (f g p): 1 0 1
+
+function value: 0.0502091  stepsize: 0.636933  
+
+used analytic gradient
+        0.00120         0.15914        -0.00022
+        0.05201         1.08981        -0.00035
+        0.10111        -0.08960        -0.00035
+        0.15100         0.32177         0.00007
+        0.20059         0.46602        -0.00054
+        0.24932        -0.85875         0.00053
+        0.29986         0.25068        -0.00040
+        0.34970        -0.16790         0.00022
+        0.39981        -0.25970        -0.00018
+        0.45047         0.56352        -0.00022
+        0.49997        -0.36597        -0.00003
+        0.55036         0.38424        -0.00016
+        0.59997         0.47629        -0.00026
+        0.64838        -0.46186        -0.00009
+        0.69713         0.76999        -0.00066
+        0.74251         0.38364        -0.00105
+        0.78319         0.88981        -0.00228
+        0.81335         0.66550        -0.00416
+        0.82440         1.28650        -0.00723
+        0.79866         2.11194        -0.01097
+        0.70966        -2.11247        -0.01026
+------------------------------------------------
+bfgsmin iteration 67  convergence (f g p): 1 0 1
+
+function value: 0.0410217  stepsize: 1.11608  
+
+used analytic gradient
+        0.00098         0.12362         0.00026
+        0.05165         0.96208        -0.00111
+        0.10076        -0.32198         0.00093
+        0.15107         0.74194        -0.00110
+        0.20006        -0.31240         0.00041
+        0.24985         0.04568        -0.00046
+        0.29946        -0.44709         0.00030
+        0.34992         0.27936        -0.00089
+        0.39963        -0.43346         0.00047
+        0.45025         0.42196        -0.00080
+        0.49994        -0.24472         0.00072
+        0.55020         0.34433         0.00015
+        0.59971         0.31455         0.00070
+        0.64829        -0.18508         0.00223
+        0.69647         0.56423         0.00220
+        0.74146         0.50833         0.00430
+        0.78091         0.72346         0.00725
+        0.80919         0.42031         0.01403
+        0.81717         0.54604         0.02587
+        0.78769        -0.24633         0.04844
+        0.69940        -0.46521         0.07840
+------------------------------------------------
+bfgsmin iteration 68  convergence (f g p): 1 0 1
+
+function value: 0.0312038  stepsize: 0.930256  
+
+used analytic gradient
+        0.00124         0.77664        -0.00123
+        0.05054        -0.63082         0.00015
+        0.10170         1.49934        -0.00184
+        0.14997        -0.89858         0.00031
+        0.20047         0.72381        -0.00048
+        0.24939        -0.70214         0.00022
+        0.29977         0.39727        -0.00001
+        0.34903        -0.92275         0.00059
+        0.40011         0.71902        -0.00082
+        0.44945        -0.85937         0.00085
+        0.50066         0.73732        -0.00093
+        0.55036        -0.07266         0.00006
+        0.60042         0.06720         0.00016
+        0.65052         0.88727        -0.00065
+        0.69867         0.15221         0.00038
+        0.74576         1.02201        -0.00071
+        0.78816         0.58029         0.00041
+        0.82322         0.90270         0.00133
+        0.84304         0.26488         0.00538
+        0.83613         1.76795         0.01221
+        0.77780        -1.48589         0.02949
+------------------------------------------------
+bfgsmin iteration 69  convergence (f g p): 1 0 1
+
+function value: 0.0170922  stepsize: 0.822259  
+
+used analytic gradient
+        0.00001        -0.27120        -0.00005
+        0.05069         0.74728        -0.00110
+        0.09986        -0.53017         0.00011
+        0.15028         0.34351        -0.00049
+        0.19999         0.02970        -0.00025
+        0.24961        -0.28733         0.00037
+        0.29976         0.06691        -0.00018
+        0.34962        -0.00518         0.00008
+        0.39929        -0.67300         0.00067
+        0.45030         0.68735        -0.00073
+        0.49973        -0.55312         0.00063
+        0.55041         0.29190        -0.00011
+        0.60057         0.45971        -0.00001
+        0.64987         0.02217         0.00077
+        0.69905         1.07750         0.00025
+        0.74505         0.00149         0.00280
+        0.78857         0.74106         0.00463
+        0.82455        -0.07162         0.01126
+        0.84841         0.06299         0.02199
+        0.84834        -0.53005         0.04147
+        0.80729         0.05430         0.06953
+------------------------------------------------
+bfgsmin iteration 70  convergence (f g p): 1 0 1
+
+function value: 0.0109398  stepsize: 0.56948  
+
+used analytic gradient
+       -0.00004         0.12700        -0.00021
+        0.04959        -0.37677        -0.00014
+        0.09997         0.21556        -0.00014
+        0.14979        -0.09217        -0.00027
+        0.19974        -0.16847         0.00028
+        0.24998         0.25007        -0.00028
+        0.29958        -0.28955         0.00044
+        0.34969        -0.12339         0.00002
+        0.39996         0.25724         0.00003
+        0.44956        -0.56480         0.00025
+        0.50036         0.41278        -0.00027
+        0.55030        -0.06181        -0.00015
+        0.60056         0.17970        -0.00040
+        0.65064         0.69973        -0.00082
+        0.69930        -0.09805        -0.00057
+        0.74786         0.85054        -0.00138
+        0.79320        -0.26430        -0.00101
+        0.83581         0.40400        -0.00189
+        0.87040         0.38395        -0.00270
+        0.88981         1.92138        -0.00428
+        0.87681        -1.28555        -0.00207
+------------------------------------------------
+bfgsmin iteration 71  convergence (f g p): 1 0 1
+
+function value: 0.00625079  stepsize: 1.3727  
+
+used analytic gradient
+       -0.00025         0.01434        -0.00035
+        0.04945        -0.37764        -0.00007
+        0.09983         0.23956        -0.00064
+        0.14952        -0.41490         0.00020
+        0.20002         0.32741        -0.00053
+        0.24970        -0.31927         0.00046
+        0.30002         0.25733        -0.00024
+        0.34971        -0.28944         0.00048
+        0.39999         0.17514        -0.00006
+        0.44982        -0.21557         0.00020
+        0.50009         0.10403        -0.00009
+        0.55015         0.05278        -0.00013
+        0.60016         0.16733        -0.00026
+        0.64983         0.26738        -0.00022
+        0.69874         0.21239         0.00008
+        0.74648         0.10443         0.00128
+        0.79219         0.03507         0.00336
+        0.83392         0.02311         0.00776
+        0.86770         0.21073         0.01570
+        0.88553        -0.14052         0.03114
+        0.87475        -0.09221         0.05597
+------------------------------------------------
+bfgsmin iteration 72  convergence (f g p): 1 0 1
+
+function value: 0.00340775  stepsize: 0.771117  
+
+used analytic gradient
+       -0.00061        -0.23907         0.00019
+        0.04938        -0.05287        -0.00003
+        0.09919        -0.45729         0.00036
+        0.14973         0.25784        -0.00021
+        0.19949        -0.46347         0.00029
+        0.25015         0.44257        -0.00023
+        0.29978        -0.35507         0.00026
+        0.35020         0.30983        -0.00017
+        0.39993        -0.15316         0.00023
+        0.45001         0.03929        -0.00010
+        0.50000        -0.01159         0.00006
+        0.55003         0.06731        -0.00024
+        0.59990         0.04168        -0.00029
+        0.64961         0.11921        -0.00033
+        0.69882        -0.12599        -0.00035
+        0.74776         0.00441        -0.00015
+        0.79555        -0.22504        -0.00008
+        0.84168         0.10920        -0.00001
+        0.88340         0.11829         0.00030
+        0.91667         1.29378         0.00103
+        0.93072        -0.74535         0.00540
+------------------------------------------------
+bfgsmin iteration 73  convergence (f g p): 1 0 1
+
+function value: 0.00173095  stepsize: 0.998157  
+
+used analytic gradient
+       -0.00042        -0.07427         0.00004
+        0.04935        -0.30189         0.00031
+        0.09954        -0.00517        -0.00007
+        0.14952        -0.21046         0.00023
+        0.19978         0.00566        -0.00007
+        0.24992        -0.01376         0.00007
+        0.30005         0.07018        -0.00009
+        0.35002        -0.05997         0.00015
+        0.40016         0.18998        -0.00019
+        0.44991        -0.18026         0.00023
+        0.50006         0.18060        -0.00023
+        0.54979        -0.07636         0.00003
+        0.59960        -0.02249        -0.00006
+        0.64927         0.04315        -0.00008
+        0.69847        -0.28593         0.00048
+        0.74761         0.03064         0.00059
+        0.79547        -0.24175         0.00180
+        0.84167         0.01041         0.00330
+        0.88370         0.01321         0.00672
+        0.91769         0.06373         0.01309
+        0.93611        -0.06318         0.02427
+------------------------------------------------
+bfgsmin iteration 74  convergence (f g p): 1 0 1
+
+function value: 0.00106288  stepsize: 1.11747  
+
+used analytic gradient
+       -0.00038        -0.16350         0.00038
+        0.04966         0.01700         0.00031
+        0.09947        -0.29303         0.00048
+        0.14976         0.08008         0.00018
+        0.19971        -0.18276         0.00019
+        0.24999         0.11702        -0.00003
+        0.29996        -0.10516         0.00002
+        0.35017         0.19800        -0.00018
+        0.39997        -0.15134         0.00011
+        0.45014         0.21866        -0.00024
+        0.49983        -0.15369         0.00010
+        0.54982         0.07022        -0.00017
+        0.59954        -0.06211        -0.00003
+        0.64919        -0.20952         0.00026
+        0.69895        -0.00714         0.00022
+        0.74820        -0.29050         0.00093
+        0.79727         0.00229         0.00113
+        0.84497        -0.10510         0.00243
+        0.89042         0.12964         0.00429
+        0.93078         0.53661         0.00863
+        0.96039        -0.30841         0.01861
+------------------------------------------------
+bfgsmin iteration 75  convergence (f g p): 1 0 1
+
+function value: 0.00029829  stepsize: 0.916043  
+
+used analytic gradient
+        0.00000         0.01563         0.00010
+        0.04996        -0.01719         0.00015
+        0.09995        -0.00586         0.00013
+        0.14993        -0.01028         0.00010
+        0.19990        -0.05644         0.00009
+        0.24996         0.00892         0.00001
+        0.29998        -0.00378        -0.00001
+        0.34999        -0.03205         0.00000
+        0.40008         0.12533        -0.00012
+        0.44990        -0.10330         0.00005
+        0.49993         0.10872        -0.00011
+        0.54965        -0.12952         0.00010
+        0.59951        -0.12585         0.00015
+        0.64945        -0.02528         0.00016
+        0.69917        -0.26038         0.00050
+        0.74913         0.09802         0.00046
+        0.79840        -0.21068         0.00118
+        0.84740         0.15004         0.00179
+        0.89471        -0.01319         0.00370
+        0.93941        -0.05330         0.00721
+        0.97900         0.01218         0.01380
+------------------------------------------------
+bfgsmin iteration 76  convergence (f g p): 1 0 1
+
+function value: 0.000101933  stepsize: 1.13817  
+
+used analytic gradient
+        0.00010         0.03644        -0.00001
+        0.05011         0.03559         0.00003
+        0.10009         0.02823         0.00001
+        0.15003         0.00221         0.00006
+        0.19999        -0.00665         0.00002
+        0.24997        -0.01504         0.00004
+        0.29996        -0.02322         0.00001
+        0.35000         0.02744        -0.00003
+        0.39996        -0.02263        -0.00001
+        0.44996         0.04347        -0.00005
+        0.49982        -0.05832         0.00005
+        0.54975        -0.04776         0.00009
+        0.59966        -0.08265         0.00015
+        0.64961        -0.12218         0.00021
+        0.69967        -0.01221         0.00013
+        0.74959        -0.10802         0.00024
+        0.79958         0.06541         0.00010
+        0.84919        -0.00821         0.00024
+        0.89841         0.08577         0.00034
+        0.94663         0.13486         0.00083
+        0.99280        -0.09176         0.00222
+------------------------------------------------
+bfgsmin iteration 77  convergence (f g p): 1 0 1
+
+function value: 3.50594e-05  stepsize: 1.04079  
+
+used analytic gradient
+        0.00009         0.02084        -0.00003
+        0.05014         0.06298        -0.00007
+        0.10009        -0.00011         0.00001
+        0.15010         0.05112        -0.00004
+        0.20002        -0.02356         0.00004
+        0.25001         0.01356        -0.00000
+        0.29997        -0.01802         0.00002
+        0.34996        -0.00327        -0.00001
+        0.39995        -0.00382        -0.00001
+        0.44991        -0.01843         0.00001
+        0.49987        -0.02505         0.00004
+        0.54983        -0.03886         0.00009
+        0.59981        -0.05491         0.00013
+        0.64982        -0.01931         0.00011
+        0.69980        -0.06345         0.00017
+        0.74983         0.04016         0.00010
+        0.79968        -0.02912         0.00022
+        0.84943         0.05871         0.00028
+        0.89876        -0.00022         0.00066
+        0.94745        -0.06148         0.00139
+        0.99502         0.02216         0.00258
+------------------------------------------------
+bfgsmin iteration 78  convergence (f g p): 1 0 1
+
+function value: 1.14112e-05  stepsize: 1.02974  
+
+used analytic gradient
+        0.00007         0.02707        -0.00005
+        0.05007         0.00360        -0.00003
+        0.10010         0.04928        -0.00007
+        0.15005        -0.00879        -0.00000
+        0.20006         0.03468        -0.00003
+        0.25001        -0.01478         0.00002
+        0.30000         0.00911        -0.00000
+        0.34996        -0.01607         0.00002
+        0.39994        -0.01093         0.00001
+        0.44992        -0.02401         0.00004
+        0.49992        -0.01834         0.00004
+        0.54992        -0.02407         0.00007
+        0.59994        -0.00263         0.00006
+        0.64994        -0.02582         0.00008
+        0.69997         0.02392         0.00003
+        0.74993        -0.02055         0.00004
+        0.79990         0.04027        -0.00001
+        0.84971        -0.01225         0.00004
+        0.89942        -0.00374         0.00010
+        0.94884         0.02867         0.00019
+        0.99760        -0.01539         0.00044
+------------------------------------------------
+bfgsmin iteration 79  convergence (f g p): 1 0 1
+
+function value: 4.23169e-06  stepsize: 1.05364  
+
+used analytic gradient
+        0.00002         0.00131        -0.00002
+        0.05004         0.01752        -0.00003
+        0.10003        -0.00227        -0.00002
+        0.15005         0.02625        -0.00004
+        0.20003        -0.00319        -0.00001
+        0.25003         0.01965        -0.00002
+        0.29999        -0.01185         0.00001
+        0.34998         0.00350         0.00000
+        0.39995        -0.02608         0.00004
+        0.44996        -0.00434         0.00002
+        0.49996        -0.02106         0.00004
+        0.54999         0.00580         0.00001
+        0.60000        -0.00327         0.00002
+        0.65001         0.01505        -0.00000
+        0.70000         0.00865        -0.00000
+        0.74997         0.01105        -0.00001
+        0.79989         0.00255         0.00000
+        0.84975        -0.01075         0.00003
+        0.89951        -0.00047         0.00005
+        0.94903         0.00703         0.00008
+        0.99804        -0.00408         0.00016
+------------------------------------------------
+bfgsmin iteration 80  convergence (f g p): 1 0 1
+
+function value: 1.96835e-06  stepsize: 1.17434  
+
+used analytic gradient
+        0.00001         0.00234        -0.00001
+        0.05001        -0.00392        -0.00001
+        0.10002         0.01042        -0.00002
+        0.15001        -0.00338        -0.00001
+        0.20002         0.01530        -0.00002
+        0.25001        -0.00488        -0.00000
+        0.30000         0.00923        -0.00000
+        0.34998        -0.01336         0.00002
+        0.39998         0.00114         0.00001
+        0.44998        -0.01455         0.00003
+        0.49999         0.00398         0.00001
+        0.55000        -0.00549         0.00001
+        0.60001         0.01062        -0.00000
+        0.65001         0.00545        -0.00001
+        0.70000         0.01028        -0.00001
+        0.74996         0.00350        -0.00001
+        0.79989        -0.00553         0.00001
+        0.84978        -0.00012         0.00000
+        0.89957         0.00567        -0.00000
+        0.94911         0.00563         0.00000
+        0.99820        -0.00539         0.00002
+------------------------------------------------
+bfgsmin iteration 81  convergence (f g p): 1 0 1
+
+function value: 1.23802e-06  stepsize: 1.01532  
+
+used analytic gradient
+       -0.00000        -0.00109         0.00000
+        0.04999        -0.00414         0.00000
+        0.10000         0.00140        -0.00000
+        0.15000        -0.00159         0.00000
+        0.20000         0.00245        -0.00000
+        0.25000         0.00163        -0.00000
+        0.30000        -0.00137         0.00000
+        0.35000         0.00102        -0.00000
+        0.40000        -0.00320         0.00000
+        0.45000         0.00211        -0.00000
+        0.50001        -0.00046        -0.00000
+        0.55001         0.00418        -0.00001
+        0.60001         0.00521        -0.00001
+        0.65000         0.00356        -0.00001
+        0.69999         0.00676        -0.00001
+        0.74995        -0.00547         0.00000
+        0.79990         0.00327        -0.00000
+        0.84979         0.00094        -0.00000
+        0.89957         0.00263        -0.00001
+        0.94912         0.00004        -0.00001
+        0.99823        -0.00216        -0.00002
+------------------------------------------------
+bfgsmin iteration 82  convergence (f g p): 1 0 1
+
+function value: 1.12478e-06  stepsize: 1.62417  
+
+used analytic gradient
+       -0.00000        -0.00106         0.00000
+        0.05000        -0.00034         0.00000
+        0.10000        -0.00092         0.00000
+        0.15000         0.00022        -0.00000
+        0.20000         0.00061        -0.00000
+        0.25000        -0.00074        -0.00000
+        0.30000         0.00139        -0.00000
+        0.35000        -0.00188         0.00000
+        0.40000         0.00204        -0.00000
+        0.45000        -0.00135         0.00000
+        0.50000         0.00265        -0.00000
+        0.55000         0.00104        -0.00000
+        0.60000         0.00257        -0.00001
+        0.64999         0.00292        -0.00001
+        0.69998        -0.00099        -0.00000
+        0.74995         0.00253        -0.00000
+        0.79989        -0.00023        -0.00000
+        0.84978         0.00286        -0.00000
+        0.89956         0.00025        -0.00000
+        0.94911         0.00051        -0.00000
+        0.99821        -0.00195        -0.00000
+------------------------------------------------
+bfgsmin iteration 83  convergence (f g p): 1 0 1
+
+function value: 1.08872e-06  stepsize: 1.2166  
+
+used analytic gradient
+       -0.00000         0.00014         0.00000
+        0.05000        -0.00065         0.00000
+        0.10000         0.00040         0.00000
+        0.15000        -0.00109         0.00000
+        0.20000         0.00003         0.00000
+        0.25000        -0.00073         0.00000
+        0.30000         0.00015        -0.00000
+        0.35000         0.00060        -0.00000
+        0.40000         0.00047        -0.00000
+        0.45000         0.00115        -0.00000
+        0.50000         0.00101        -0.00000
+        0.55000         0.00068        -0.00000
+        0.60000         0.00105        -0.00000
+        0.64999        -0.00102         0.00000
+        0.69998         0.00106        -0.00000
+        0.74995         0.00081        -0.00000
+        0.79989         0.00163        -0.00000
+        0.84978        -0.00023        -0.00000
+        0.89956         0.00103        -0.00000
+        0.94910        -0.00099         0.00000
+        0.99820        -0.00107         0.00000
+------------------------------------------------
+bfgsmin iteration 84  convergence (f g p): 1 0 1
+
+function value: 1.07552e-06  stepsize: 1.60465  
+
+used analytic gradient
+        0.00000         0.00004         0.00000
+        0.05000         0.00028        -0.00000
+        0.10000        -0.00027         0.00000
+        0.15000         0.00047        -0.00000
+        0.20000        -0.00056         0.00000
+        0.25000         0.00076        -0.00000
+        0.30000        -0.00058         0.00000
+        0.35000         0.00059        -0.00000
+        0.40000        -0.00042        -0.00000
+        0.45000         0.00073        -0.00000
+        0.50000        -0.00030        -0.00000
+        0.55000         0.00089        -0.00000
+        0.59999        -0.00059         0.00000
+        0.64999         0.00106        -0.00000
+        0.69998         0.00107        -0.00000
+        0.74995         0.00022        -0.00000
+        0.79989         0.00013        -0.00000
+        0.84978         0.00057        -0.00000
+        0.89955        -0.00052         0.00000
+        0.94911         0.00023         0.00000
+        0.99820        -0.00129         0.00000
+------------------------------------------------
+bfgsmin iteration 85  convergence (f g p): 1 0 1
+
+function value: 1.06848e-06  stepsize: 1.88104  
+
+used analytic gradient
+        0.00000         0.00028        -0.00000
+        0.05000        -0.00033         0.00000
+        0.10000         0.00070        -0.00000
+        0.15000        -0.00042         0.00000
+        0.20000         0.00082        -0.00000
+        0.25000        -0.00009        -0.00000
+        0.30000         0.00026        -0.00000
+        0.35000        -0.00020        -0.00000
+        0.40000        -0.00020        -0.00000
+        0.45000        -0.00021        -0.00000
+        0.50000        -0.00007        -0.00000
+        0.55000        -0.00053        -0.00000
+        0.59999         0.00067        -0.00000
+        0.64999         0.00029        -0.00000
+        0.69997         0.00144        -0.00000
+        0.74994        -0.00084         0.00000
+        0.79989         0.00028        -0.00000
+        0.84978        -0.00073         0.00000
+        0.89955         0.00028         0.00000
+        0.94911        -0.00082         0.00000
+        0.99821        -0.00078         0.00001
+------------------------------------------------
+bfgsmin iteration 86  convergence (f g p): 1 0 1
+
+function value: 1.06108e-06  stepsize: 2.08078  
+
+used analytic gradient
+       -0.00000        -0.00022        -0.00000
+        0.05000         0.00020        -0.00000
+        0.10000        -0.00014        -0.00000
+        0.15000         0.00039        -0.00000
+        0.20000         0.00043        -0.00000
+        0.25000        -0.00023        -0.00000
+        0.30000         0.00053        -0.00000
+        0.35000        -0.00058         0.00000
+        0.40000         0.00019        -0.00000
+        0.45000        -0.00090         0.00000
+        0.50000        -0.00001        -0.00000
+        0.55000        -0.00042         0.00000
+        0.59999         0.00104        -0.00000
+        0.64999         0.00019        -0.00000
+        0.69997        -0.00148        -0.00000
+        0.74995         0.00123        -0.00000
+        0.79989        -0.00099         0.00000
+        0.84978        -0.00037         0.00000
+        0.89956        -0.00020         0.00000
+        0.94911        -0.00098         0.00001
+        0.99822        -0.00056         0.00001
+------------------------------------------------
+bfgsmin iteration 87  convergence (f g p): 1 0 1
+
+function value: 1.04988e-06  stepsize: 3.88599  
+
+used analytic gradient
+       -0.00000        -0.00028        -0.00000
+        0.05000        -0.00034        -0.00000
+        0.10000         0.00008        -0.00000
+        0.15000         0.00013        -0.00000
+        0.20000        -0.00018        -0.00000
+        0.25000         0.00013        -0.00000
+        0.30000        -0.00041        -0.00000
+        0.35000         0.00045        -0.00000
+        0.40000        -0.00065         0.00000
+        0.45000        -0.00006         0.00000
+        0.50000        -0.00063         0.00000
+        0.55000         0.00092        -0.00000
+        0.59999         0.00008        -0.00000
+        0.64998        -0.00177        -0.00000
+        0.69997        -0.00112        -0.00000
+        0.74994        -0.00108        -0.00000
+        0.79989         0.00061        -0.00000
+        0.84978        -0.00092         0.00000
+        0.89956        -0.00092         0.00001
+        0.94912        -0.00053         0.00001
+        0.99823        -0.00054         0.00003
+------------------------------------------------
+bfgsmin iteration 88  convergence (f g p): 1 0 1
+
+function value: 1.03195e-06  stepsize: 5.32276  
+
+used analytic gradient
+       -0.00000        -0.00021         0.00000
+        0.05000        -0.00020        -0.00000
+        0.10000        -0.00035        -0.00000
+        0.15000        -0.00069        -0.00000
+        0.20000        -0.00034        -0.00000
+        0.25000        -0.00059        -0.00000
+        0.30000         0.00031        -0.00000
+        0.35000        -0.00092         0.00000
+        0.40000         0.00086         0.00000
+        0.45000        -0.00036         0.00000
+        0.50000         0.00174        -0.00000
+        0.54999        -0.00069        -0.00000
+        0.59999        -0.00279         0.00000
+        0.64998        -0.00042        -0.00000
+        0.69997        -0.00133         0.00000
+        0.74994        -0.00128         0.00000
+        0.79989        -0.00181         0.00000
+        0.84978         0.00101         0.00001
+        0.89957        -0.00103         0.00003
+        0.94913         0.00000         0.00004
+        0.99826        -0.00084         0.00008
+------------------------------------------------
+bfgsmin iteration 89  convergence (f g p): 1 0 1
+
+function value: 9.79829e-07  stepsize: 5.1304  
+
+used analytic gradient
+       -0.00000         0.00023         0.00000
+        0.05000         0.00038        -0.00000
+        0.10000        -0.00198        -0.00000
+        0.15000        -0.00093        -0.00001
+        0.20000        -0.00180        -0.00001
+        0.25000        -0.00010        -0.00001
+        0.30000        -0.00159        -0.00000
+        0.35000         0.00112         0.00000
+        0.40000        -0.00061         0.00001
+        0.45000         0.00486        -0.00000
+        0.50000        -0.00083        -0.00001
+        0.54999        -0.00413         0.00000
+        0.59999        -0.00084         0.00001
+        0.64998        -0.00165         0.00001
+        0.69997         0.00013         0.00001
+        0.74994        -0.00088         0.00002
+        0.79989        -0.00184         0.00004
+        0.84979        -0.00251         0.00006
+        0.89959         0.00481         0.00011
+        0.94917        -0.00168         0.00022
+        0.99834        -0.00099         0.00043
+------------------------------------------------
+bfgsmin iteration 90  convergence (f g p): 1 0 1
+
+function value: 7.12702e-07  stepsize: 2.23805  
+
+used analytic gradient
+       -0.00000         0.00234        -0.00001
+        0.04999        -0.00301         0.00000
+        0.09999        -0.00018        -0.00000
+        0.14999        -0.00555         0.00000
+        0.19999        -0.00173        -0.00000
+        0.24999        -0.00443         0.00000
+        0.30000        -0.00055         0.00000
+        0.35000         0.00087         0.00000
+        0.40001         0.00832        -0.00001
+        0.45000         0.00129        -0.00001
+        0.49999        -0.00813         0.00001
+        0.54999        -0.00135         0.00000
+        0.59999         0.00044         0.00000
+        0.64999        -0.00108         0.00001
+        0.69998         0.00163         0.00001
+        0.74997         0.00109         0.00003
+        0.79993         0.00171         0.00004
+        0.84985         0.00006         0.00008
+        0.89970        -0.00236         0.00015
+        0.94940         0.00814         0.00029
+        0.99877        -0.00445         0.00059
+------------------------------------------------
+bfgsmin iteration 91  convergence (f g p): 1 0 1
+
+function value: 3.5481e-07  stepsize: 1.84728  
+
+used analytic gradient
+       -0.00001        -0.00199         0.00000
+        0.04999         0.00150        -0.00000
+        0.09999        -0.00545         0.00001
+        0.14999        -0.00076         0.00000
+        0.19998        -0.00700         0.00001
+        0.24999        -0.00212         0.00000
+        0.30000        -0.00116         0.00000
+        0.35001         0.00556        -0.00001
+        0.40000         0.00309        -0.00001
+        0.44999        -0.00615         0.00001
+        0.50000         0.00144         0.00000
+        0.55000        -0.00204         0.00001
+        0.60000        -0.00178         0.00001
+        0.65000         0.00424         0.00000
+        0.70000        -0.00013         0.00001
+        0.74999         0.00409         0.00001
+        0.79997         0.00310         0.00002
+        0.84993         0.00303         0.00004
+        0.89985         0.00133         0.00008
+        0.94968        -0.00390         0.00017
+        0.99937         0.00048         0.00034
+------------------------------------------------
+bfgsmin iteration 92  convergence (f g p): 1 0 1
+
+function value: 1.5078e-07  stepsize: 1.0385  
+
+used analytic gradient
+       -0.00000        -0.00090         0.00000
+        0.04999        -0.00329         0.00000
+        0.09999         0.00104         0.00000
+        0.14999        -0.00492         0.00001
+        0.19999        -0.00036         0.00000
+        0.24999        -0.00273         0.00001
+        0.30000         0.00278        -0.00000
+        0.35000         0.00028        -0.00000
+        0.39999        -0.00515         0.00000
+        0.45000         0.00218        -0.00000
+        0.50000        -0.00063         0.00000
+        0.55000         0.00094         0.00000
+        0.60000         0.00116         0.00000
+        0.65000        -0.00078         0.00000
+        0.70001         0.00442        -0.00000
+        0.75000         0.00106         0.00000
+        0.79999         0.00350         0.00000
+        0.84997         0.00296         0.00001
+        0.89993         0.00024         0.00004
+        0.94985        -0.00303         0.00008
+        0.99971         0.00057         0.00016
+------------------------------------------------
+bfgsmin iteration 93  convergence (f g p): 1 0 1
+
+function value: 5.51094e-08  stepsize: 1.32816  
+
+used analytic gradient
+       -0.00000        -0.00124         0.00000
+        0.05000        -0.00086         0.00000
+        0.10000        -0.00193         0.00000
+        0.15000         0.00039         0.00000
+        0.20000        -0.00099         0.00000
+        0.25000         0.00215        -0.00000
+        0.30000        -0.00060        -0.00000
+        0.34999        -0.00390         0.00000
+        0.40000        -0.00059         0.00000
+        0.45000        -0.00014         0.00000
+        0.50000         0.00109        -0.00000
+        0.55000         0.00130        -0.00000
+        0.60000         0.00099        -0.00000
+        0.65000         0.00105        -0.00000
+        0.70000         0.00055        -0.00000
+        0.75000         0.00255        -0.00000
+        0.80000         0.00103         0.00000
+        0.84999         0.00088         0.00000
+        0.89997        -0.00059         0.00002
+        0.94993         0.00193         0.00003
+        0.99986        -0.00118         0.00007
+------------------------------------------------
+bfgsmin iteration 94  convergence (f g p): 1 0 1
+
+function value: 1.54269e-08  stepsize: 1.12755  
+
+used analytic gradient
+       -0.00000        -0.00030         0.00000
+        0.05000        -0.00050         0.00000
+        0.10000        -0.00025         0.00000
+        0.15000        -0.00001         0.00000
+        0.20000         0.00211        -0.00000
+        0.25000        -0.00055         0.00000
+        0.30000        -0.00218         0.00000
+        0.35000         0.00024         0.00000
+        0.40000        -0.00044         0.00000
+        0.45000        -0.00059         0.00000
+        0.50000         0.00080        -0.00000
+        0.55000         0.00064        -0.00000
+        0.60000         0.00081        -0.00000
+        0.65000         0.00112        -0.00000
+        0.70000         0.00043        -0.00000
+        0.75000        -0.00007        -0.00000
+        0.80000         0.00059        -0.00000
+        0.84999        -0.00116         0.00000
+        0.89998         0.00200         0.00000
+        0.94997        -0.00057         0.00001
+        0.99993        -0.00017         0.00002
+------------------------------------------------
+bfgsmin iteration 95  convergence (f g p): 1 0 1
+
+function value: 3.73216e-09  stepsize: 1.12294  
+
+used analytic gradient
+        0.00000         0.00003         0.00000
+        0.05000        -0.00007         0.00000
+        0.10000         0.00031        -0.00000
+        0.15000         0.00075        -0.00000
+        0.20000         0.00026        -0.00000
+        0.25000        -0.00073         0.00000
+        0.30000        -0.00019         0.00000
+        0.35000        -0.00007         0.00000
+        0.40000        -0.00006         0.00000
+        0.45000         0.00017        -0.00000
+        0.50000        -0.00022         0.00000
+        0.55000         0.00045        -0.00000
+        0.60000         0.00063        -0.00000
+        0.65000         0.00039        -0.00000
+        0.70000         0.00039        -0.00000
+        0.75000        -0.00028         0.00000
+        0.80000        -0.00082         0.00000
+        0.84999         0.00085        -0.00000
+        0.89999        -0.00044         0.00000
+        0.94998         0.00024         0.00000
+        0.99995        -0.00010         0.00000
+------------------------------------------------
+bfgsmin iteration 96  convergence (f g p): 1 0 1
+
+function value: 1.38589e-09  stepsize: 1.04438  
+
+used analytic gradient
+        0.00000         0.00003         0.00000
+        0.05000         0.00025        -0.00000
+        0.10000         0.00038        -0.00000
+        0.15000         0.00029        -0.00000
+        0.20000        -0.00040         0.00000
+        0.25000         0.00011         0.00000
+        0.30000         0.00006         0.00000
+        0.35000        -0.00016         0.00000
+        0.40000         0.00013        -0.00000
+        0.45000        -0.00000        -0.00000
+        0.50000         0.00012        -0.00000
+        0.55000         0.00008        -0.00000
+        0.60000         0.00025        -0.00000
+        0.65000         0.00026        -0.00000
+        0.70000        -0.00002        -0.00000
+        0.75000        -0.00020         0.00000
+        0.80000         0.00012         0.00000
+        0.84999        -0.00032         0.00000
+        0.89999         0.00009         0.00000
+        0.94998        -0.00013         0.00000
+        0.99996         0.00005         0.00000
+------------------------------------------------
+bfgsmin iteration 97  convergence (f g p): 1 0 1
+
+function value: 8.56497e-10  stepsize: 1.37931  
+
+used analytic gradient
+        0.00000         0.00013        -0.00000
+        0.05000         0.00022        -0.00000
+        0.10000         0.00019        -0.00000
+        0.15000        -0.00019         0.00000
+        0.20000         0.00015        -0.00000
+        0.25000        -0.00001         0.00000
+        0.30000         0.00002         0.00000
+        0.35000         0.00011        -0.00000
+        0.40000        -0.00007         0.00000
+        0.45000         0.00008        -0.00000
+        0.50000         0.00008        -0.00000
+        0.55000         0.00008        -0.00000
+        0.60000         0.00002        -0.00000
+        0.65000         0.00002        -0.00000
+        0.70000        -0.00006         0.00000
+        0.75000         0.00012        -0.00000
+        0.80000        -0.00008         0.00000
+        0.84999        -0.00008         0.00000
+        0.89999        -0.00013         0.00000
+        0.94998         0.00002         0.00000
+        0.99996         0.00001         0.00000
+------------------------------------------------
+bfgsmin iteration 98  convergence (f g p): 1 0 1
+
+function value: 6.91006e-10  stepsize: 1.34714  
+
+used analytic gradient
+        0.00000         0.00014        -0.00000
+        0.05000         0.00008        -0.00000
+        0.10000        -0.00011         0.00000
+        0.15000         0.00010        -0.00000
+        0.20000         0.00001        -0.00000
+        0.25000         0.00006        -0.00000
+        0.30000         0.00004        -0.00000
+        0.35000         0.00002        -0.00000
+        0.40000         0.00007        -0.00000
+        0.45000         0.00001        -0.00000
+        0.50000         0.00005        -0.00000
+        0.55000         0.00003        -0.00000
+        0.60000        -0.00002        -0.00000
+        0.65000        -0.00010         0.00000
+        0.70000         0.00008        -0.00000
+        0.75000        -0.00002        -0.00000
+        0.80000        -0.00001         0.00000
+        0.84999        -0.00001         0.00000
+        0.89999        -0.00006         0.00000
+        0.94998        -0.00003         0.00000
+        0.99996         0.00000         0.00000
+------------------------------------------------
+bfgsmin iteration 99  convergence (f g p): 1 0 1
+
+function value: 6.36262e-10  stepsize: 1.26274  
+
+used analytic gradient
+        0.00000         0.00004        -0.00000
+        0.05000        -0.00006         0.00000
+        0.10000         0.00003        -0.00000
+        0.15000         0.00001        -0.00000
+        0.20000         0.00004        -0.00000
+        0.25000         0.00003        -0.00000
+        0.30000         0.00003        -0.00000
+        0.35000         0.00002        -0.00000
+        0.40000         0.00004        -0.00000
+        0.45000         0.00003        -0.00000
+        0.50000        -0.00001        -0.00000
+        0.55000        -0.00001        -0.00000
+        0.60000        -0.00005         0.00000
+        0.65000         0.00002        -0.00000
+        0.70000        -0.00004         0.00000
+        0.75000        -0.00000         0.00000
+        0.80000         0.00001        -0.00000
+        0.84999        -0.00001         0.00000
+        0.89999         0.00002         0.00000
+        0.94998        -0.00004         0.00000
+        0.99996        -0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 100  convergence (f g p): 1 0 1
+
+function value: 6.24825e-10  stepsize: 2.08371  
+
+used analytic gradient
+       -0.00000        -0.00005         0.00000
+        0.05000         0.00002        -0.00000
+        0.10000         0.00000        -0.00000
+        0.15000         0.00001        -0.00000
+        0.20000         0.00002        -0.00000
+        0.25000         0.00002        -0.00000
+        0.30000         0.00002        -0.00000
+        0.35000         0.00003        -0.00000
+        0.40000         0.00001        -0.00000
+        0.45000         0.00001        -0.00000
+        0.50000         0.00000        -0.00000
+        0.55000        -0.00003         0.00000
+        0.60000         0.00001         0.00000
+        0.65000        -0.00002         0.00000
+        0.70000        -0.00001         0.00000
+        0.75000        -0.00001         0.00000
+        0.80000        -0.00000         0.00000
+        0.84999         0.00002        -0.00000
+        0.89999        -0.00001         0.00000
+        0.94998         0.00001         0.00000
+        0.99996        -0.00003         0.00000
+------------------------------------------------
+bfgsmin iteration 101  convergence (f g p): 1 0 1
+
+function value: 6.16161e-10  stepsize: 1.18206  
+
+used analytic gradient
+       -0.00000        -0.00002         0.00000
+        0.05000        -0.00000        -0.00000
+        0.10000        -0.00000        -0.00000
+        0.15000         0.00000        -0.00000
+        0.20000        -0.00001        -0.00000
+        0.25000        -0.00000        -0.00000
+        0.30000         0.00001        -0.00000
+        0.35000        -0.00000        -0.00000
+        0.40000         0.00000        -0.00000
+        0.45000        -0.00002         0.00000
+        0.50000        -0.00001         0.00000
+        0.55000         0.00003        -0.00000
+        0.60000        -0.00001         0.00000
+        0.65000        -0.00001         0.00000
+        0.70000         0.00001         0.00000
+        0.75000        -0.00000         0.00000
+        0.80000         0.00000         0.00000
+        0.84999        -0.00001         0.00000
+        0.89999         0.00001        -0.00000
+        0.94998        -0.00001         0.00000
+        0.99996        -0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 102  convergence (f g p): 1 0 1
+
+function value: 6.1409e-10  stepsize: 3.50012  
+
+used analytic gradient
+       -0.00000         0.00001         0.00000
+        0.05000        -0.00001         0.00000
+        0.10000         0.00000        -0.00000
+        0.15000        -0.00001         0.00000
+        0.20000        -0.00000        -0.00000
+        0.25000        -0.00000        -0.00000
+        0.30000        -0.00001        -0.00000
+        0.35000         0.00000        -0.00000
+        0.40000        -0.00001         0.00000
+        0.45000        -0.00001         0.00000
+        0.50000         0.00000        -0.00000
+        0.55000        -0.00000        -0.00000
+        0.60000         0.00001        -0.00000
+        0.65000         0.00000         0.00000
+        0.70000        -0.00000         0.00000
+        0.75000         0.00001        -0.00000
+        0.80000        -0.00001         0.00000
+        0.84999         0.00000         0.00000
+        0.89999        -0.00001         0.00000
+        0.94998        -0.00001         0.00000
+        0.99996        -0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 103  convergence (f g p): 1 0 1
+
+function value: 6.11911e-10  stepsize: 4.49361  
+
+used analytic gradient
+        0.00000         0.00000         0.00000
+        0.05000         0.00002         0.00000
+        0.10000        -0.00002         0.00000
+        0.15000        -0.00000         0.00000
+        0.20000         0.00000        -0.00000
+        0.25000        -0.00001        -0.00000
+        0.30000        -0.00001        -0.00000
+        0.35000        -0.00002         0.00000
+        0.40000         0.00000         0.00000
+        0.45000         0.00001         0.00000
+        0.50000        -0.00001         0.00000
+        0.55000        -0.00001         0.00000
+        0.60000         0.00000        -0.00000
+        0.65000         0.00001        -0.00000
+        0.70000         0.00001         0.00000
+        0.75000        -0.00001         0.00000
+        0.80000         0.00002         0.00000
+        0.84999        -0.00001         0.00000
+        0.89999        -0.00001         0.00000
+        0.94998        -0.00002         0.00000
+        0.99996        -0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 104  convergence (f g p): 1 0 1
+
+function value: 5.99775e-10  stepsize: 4.15156  
+
+used analytic gradient
+        0.00000         0.00003         0.00000
+        0.05000         0.00001         0.00000
+        0.10000         0.00006         0.00000
+        0.15000        -0.00003         0.00000
+        0.20000        -0.00001         0.00000
+        0.25000        -0.00001        -0.00000
+        0.30000        -0.00005         0.00000
+        0.35000        -0.00002         0.00000
+        0.40000         0.00006         0.00000
+        0.45000         0.00003         0.00000
+        0.50000        -0.00001         0.00000
+        0.55000         0.00002         0.00000
+        0.60000        -0.00004         0.00000
+        0.65000         0.00001        -0.00000
+        0.70000         0.00001         0.00000
+        0.75000         0.00003         0.00000
+        0.80000        -0.00003         0.00000
+        0.84999         0.00001         0.00000
+        0.89999        -0.00001         0.00000
+        0.94998        -0.00003         0.00000
+        0.99996        -0.00001         0.00000
+------------------------------------------------
+bfgsmin iteration 105  convergence (f g p): 1 0 1
+
+function value: 5.6718e-10  stepsize: 4.77996  
+
+used analytic gradient
+        0.00000         0.00004         0.00000
+        0.05000         0.00007         0.00000
+        0.10000         0.00004         0.00000
+        0.15000         0.00008         0.00000
+        0.20000        -0.00006         0.00000
+        0.25000        -0.00006         0.00000
+        0.30000        -0.00001         0.00000
+        0.35000         0.00008         0.00000
+        0.40000         0.00000         0.00000
+        0.45000         0.00003         0.00000
+        0.50000         0.00008         0.00000
+        0.55000        -0.00005         0.00000
+        0.60000         0.00003         0.00000
+        0.65000        -0.00008         0.00000
+        0.70000         0.00004        -0.00000
+        0.75000         0.00002         0.00000
+        0.80000         0.00001         0.00000
+        0.85000        -0.00003         0.00000
+        0.89999         0.00000         0.00000
+        0.94998         0.00001         0.00000
+        0.99996        -0.00003         0.00001
+------------------------------------------------
+bfgsmin iteration 106  convergence (f g p): 1 0 1
+
+function value: 4.66432e-10  stepsize: 1.62031  
+
+used analytic gradient
+        0.00000         0.00005        -0.00000
+        0.05000         0.00012        -0.00000
+        0.10000         0.00005        -0.00000
+        0.15000         0.00006         0.00000
+        0.20000         0.00007         0.00000
+        0.25000        -0.00010         0.00000
+        0.30000         0.00017         0.00000
+        0.35000         0.00001         0.00000
+        0.40000         0.00002         0.00000
+        0.45000         0.00008        -0.00000
+        0.50000        -0.00001         0.00000
+        0.55000         0.00012        -0.00000
+        0.60000        -0.00008         0.00000
+        0.65000         0.00010         0.00000
+        0.70000        -0.00013         0.00000
+        0.75000        -0.00002         0.00000
+        0.80000         0.00008         0.00000
+        0.85000        -0.00001         0.00000
+        0.89999        -0.00001         0.00000
+        0.94998         0.00003         0.00000
+        0.99997        -0.00004         0.00001
+------------------------------------------------
+bfgsmin iteration 107  convergence (f g p): 1 0 1
+
+function value: 3.4185e-10  stepsize: 3.91592  
+
+used analytic gradient
+        0.00000         0.00007        -0.00000
+        0.05000         0.00003        -0.00000
+        0.10000         0.00008        -0.00000
+        0.15000         0.00002        -0.00000
+        0.20000         0.00008         0.00000
+        0.25000         0.00022        -0.00000
+        0.30000         0.00003        -0.00000
+        0.35000         0.00005        -0.00000
+        0.40000         0.00006        -0.00000
+        0.45000        -0.00001        -0.00000
+        0.50000         0.00005        -0.00000
+        0.55000         0.00003        -0.00000
+        0.60000         0.00011        -0.00000
+        0.65000        -0.00002         0.00000
+        0.70000        -0.00004         0.00000
+        0.75000        -0.00008         0.00000
+        0.80000         0.00000         0.00000
+        0.85000         0.00009         0.00000
+        0.89999         0.00001         0.00000
+        0.94999        -0.00003         0.00001
+        0.99998        -0.00001         0.00001
+------------------------------------------------
+bfgsmin iteration 108  convergence (f g p): 1 0 1
+
+function value: 1.42136e-10  stepsize: 1.49851  
+
+used analytic gradient
+       -0.00000        -0.00007        -0.00000
+        0.05000         0.00000        -0.00000
+        0.10000        -0.00011         0.00000
+        0.15000         0.00010        -0.00000
+        0.20000         0.00017        -0.00000
+        0.25000         0.00005        -0.00000
+        0.30000         0.00010        -0.00000
+        0.35000         0.00003        -0.00000
+        0.40000        -0.00004        -0.00000
+        0.45000         0.00005        -0.00000
+        0.50000        -0.00002        -0.00000
+        0.55000         0.00002        -0.00000
+        0.60000        -0.00001        -0.00000
+        0.65000        -0.00000         0.00000
+        0.70000         0.00007         0.00000
+        0.75000         0.00009         0.00000
+        0.80000        -0.00012         0.00000
+        0.85000        -0.00006         0.00000
+        0.90000         0.00008         0.00000
+        0.94999        -0.00003         0.00000
+        0.99999        -0.00000         0.00001
+------------------------------------------------
+bfgsmin iteration 109  convergence (f g p): 1 0 1
+
+function value: 4.86385e-11  stepsize: 1.30001  
+
+used analytic gradient
+       -0.00000        -0.00004         0.00000
+        0.05000        -0.00013         0.00000
+        0.10000         0.00005        -0.00000
+        0.15000         0.00001        -0.00000
+        0.20000         0.00004        -0.00000
+        0.25000         0.00005        -0.00000
+        0.30000         0.00004        -0.00000
+        0.35000         0.00000        -0.00000
+        0.40000         0.00003        -0.00000
+        0.45000        -0.00004         0.00000
+        0.50000         0.00002        -0.00000
+        0.55000        -0.00002        -0.00000
+        0.60000        -0.00005         0.00000
+        0.65000         0.00008        -0.00000
+        0.70000         0.00004        -0.00000
+        0.75000        -0.00001         0.00000
+        0.80000         0.00005         0.00000
+        0.85000        -0.00006         0.00000
+        0.90000        -0.00008         0.00000
+        0.95000         0.00007         0.00000
+        1.00000        -0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 110  convergence (f g p): 1 0 1
+
+function value: 9.12869e-12  stepsize: 0.626177  
+
+used analytic gradient
+       -0.00000        -0.00005         0.00000
+        0.05000        -0.00000         0.00000
+        0.10000        -0.00001         0.00000
+        0.15000         0.00001        -0.00000
+        0.20000        -0.00003         0.00000
+        0.25000         0.00002        -0.00000
+        0.30000        -0.00001        -0.00000
+        0.35000         0.00003        -0.00000
+        0.40000        -0.00000        -0.00000
+        0.45000         0.00001        -0.00000
+        0.50000        -0.00002         0.00000
+        0.55000        -0.00003         0.00000
+        0.60000         0.00003        -0.00000
+        0.65000        -0.00000        -0.00000
+        0.70000         0.00001        -0.00000
+        0.75000         0.00001        -0.00000
+        0.80000         0.00002         0.00000
+        0.85000         0.00000         0.00000
+        0.90000        -0.00003         0.00000
+        0.95000        -0.00004         0.00000
+        1.00000         0.00002         0.00000
+------------------------------------------------
+bfgsmin iteration 111  convergence (f g p): 1 0 1
+
+function value: 2.65241e-12  stepsize: 0.871216  
+
+used analytic gradient
+       -0.00000        -0.00001         0.00000
+        0.05000        -0.00001         0.00000
+        0.10000        -0.00000         0.00000
+        0.15000        -0.00002         0.00000
+        0.20000         0.00001        -0.00000
+        0.25000        -0.00003         0.00000
+        0.30000         0.00002        -0.00000
+        0.35000        -0.00000        -0.00000
+        0.40000         0.00001        -0.00000
+        0.45000        -0.00000         0.00000
+        0.50000        -0.00001         0.00000
+        0.55000         0.00001         0.00000
+        0.60000        -0.00000        -0.00000
+        0.65000         0.00000        -0.00000
+        0.70000        -0.00000        -0.00000
+        0.75000         0.00001        -0.00000
+        0.80000         0.00000         0.00000
+        0.85000         0.00000         0.00000
+        0.90000        -0.00000         0.00000
+        0.95000        -0.00002         0.00000
+        1.00000         0.00001         0.00000
+------------------------------------------------
+bfgsmin iteration 112  convergence (f g p): 1 0 1
+
+function value: 1.37738e-12  stepsize: 1.77508  
+
+used analytic gradient
+       -0.00000         0.00000         0.00000
+        0.05000        -0.00001         0.00000
+        0.10000        -0.00001         0.00000
+        0.15000        -0.00001         0.00000
+        0.20000        -0.00001         0.00000
+        0.25000        -0.00000         0.00000
+        0.30000        -0.00001         0.00000
+        0.35000         0.00000        -0.00000
+        0.40000        -0.00000        -0.00000
+        0.45000        -0.00000         0.00000
+        0.50000         0.00000         0.00000
+        0.55000         0.00000        -0.00000
+        0.60000        -0.00000         0.00000
+        0.65000        -0.00000         0.00000
+        0.70000         0.00000        -0.00000
+        0.75000         0.00000        -0.00000
+        0.80000         0.00000        -0.00000
+        0.85000        -0.00000         0.00000
+        0.90000         0.00000         0.00000
+        0.95000         0.00000         0.00000
+        1.00000        -0.00000         0.00000
+------------------------------------------------
+bfgsmin iteration 113  convergence (f g p): 1 0 1
+
+function value: 8.55369e-13  stepsize: 1.52252  
+
+used analytic gradient
+       -0.00000        -0.00000         0.00000
+        0.05000         0.00000         0.00000
+        0.10000        -0.00000         0.00000
+        0.15000        -0.00000         0.00000
+        0.20000        -0.00000         0.00000
+        0.25000        -0.00000         0.00000
+        0.30000        -0.00000         0.00000
+        0.35000        -0.00001         0.00000
+        0.40000        -0.00000         0.00000
+        0.45000         0.00000        -0.00000
+        0.50000         0.00000        -0.00000
+        0.55000        -0.00000         0.00000
+        0.60000         0.00000        -0.00000
+        0.65000         0.00000        -0.00000
+        0.70000        -0.00000         0.00000
+        0.75000         0.00000        -0.00000
+        0.80000        -0.00000         0.00000
+        0.85000         0.00000        -0.00000
+        0.90000         0.00000        -0.00000
+        0.95000         0.00000         0.00000
+        1.00000        -0.00000         0.00000
+------------------------------------------------
+bfgsmin final results: 114 iterations
+
+function value: 6.74546e-13
+
+STRONG CONVERGENCE
+Function conv 1  Param conv 1  Gradient conv 1
+
+used analytic gradient
+          param    gradient (n)          change
+       -0.00000        -0.00000         0.00000
+        0.05000        -0.00000         0.00000
+        0.10000         0.00000        -0.00000
+        0.15000         0.00000         0.00000
+        0.20000        -0.00000         0.00000
+        0.25000        -0.00000         0.00000
+        0.30000        -0.00000         0.00000
+        0.35000        -0.00000         0.00000
+        0.40000        -0.00000         0.00000
+        0.45000        -0.00000        -0.00000
+        0.50000        -0.00000        -0.00000
+        0.55000         0.00000        -0.00000
+        0.60000        -0.00000         0.00000
+        0.65000         0.00000        -0.00000
+        0.70000        -0.00000         0.00000
+        0.75000        -0.00000         0.00000
+        0.80000         0.00000        -0.00000
+        0.85000         0.00000        -0.00000
+        0.90000         0.00000        -0.00000
+        0.95000        -0.00000         0.00000
+        1.00000        -0.00000         0.00000
+linf_norm =  0.0000014103

diff --git a/tests/optimization/bfgs_05.py b/tests/optimization/bfgs_05.py
new file mode 100755 (executable)
index 0000000..8eb3f25
--- /dev/null
+++ b/tests/optimization/bfgs_05.py
@@ -0,0 +1,52 @@
+# //-----------------------------------------------------------
+# //
+# //    Copyright (C) 2018 by the deal.II authors
+# //
+# //    This file is part of the deal.II library.
+# //
+# //    The deal.II library is free software; you can use it, redistribute
+# //    it, and/or modify it under the terms of the GNU Lesser General
+# //    Public License as published by the Free Software Foundation; either
+# //    version 2.1 of the License, or (at your option) any later version.
+# //    The full text of the license can be found in the file LICENSE.md at
+# //    the top level directory of deal.II.
+# //
+# //---------------------------------------------------------------
+
+# A companion to bfgs_05.cc which minimizes the Rosenbrok function using numpy
+
+import numpy as np
+from scipy.optimize import fmin_l_bfgs_b
+from scipy.optimize import rosen
+from scipy.optimize import rosen_der
+
+# see https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin_l_bfgs_b.html
+
+# dimension of Rosenbrok function
+dim = 20
+
+x0 = np.zeros(dim+1)
+one = np.ones(dim+1)
+location = np.zeros(dim+1)
+for i in range(dim+1):
+    location[i] = (1.*i)/dim
+
+
+def v_rosen(theta):
+    return rosen(theta - location + one)
+
+
+def g_rosen(theta):
+    return rosen_der(theta - location + one)
+
+
+x, min_val, info = fmin_l_bfgs_b(func=v_rosen,x0=x0,fprime=g_rosen,m=3, factr=10)
+dx = x - location
+
+print "{0} iterations".format(info['nit'])
+print "function value: {0}".format(min_val)
+print "linf_norm =     {0}".format(np.linalg.norm(dx,ord=np.inf))
+print "Gradient noorm: {0}".format(np.linalg.norm(g_rosen(x)))
+print "function calls: {0}".format(info['funcalls'])
+print "Solution:"
+print x

diff --git a/tests/optimization/bfgs_05b.cc b/tests/optimization/bfgs_05b.cc
new file mode 100644 (file)
index 0000000..47efae0
--- /dev/null
+++ b/tests/optimization/bfgs_05b.cc
@@ -0,0 +1,138 @@
+//-----------------------------------------------------------
+//
+//    Copyright (C) 2018 by the deal.II authors
+//
+//    This file is part of the deal.II library.
+//
+//    The deal.II library is free software; you can use it, redistribute
+//    it, and/or modify it under the terms of the GNU Lesser General
+//    Public License as published by the Free Software Foundation; either
+//    version 2.1 of the License, or (at your option) any later version.
+//    The full text of the license can be found in the file LICENSE.md at
+//    the top level directory of deal.II.
+//
+//---------------------------------------------------------------
+
+// test limited memory BFGS with Rosenbrock function.
+// same as bfgs_05 but tests with default line search function which is the
+// same asin bfgs_05.cc and therefore the number of iterations until convergence
+// is exactly the same.
+
+#include <deal.II/base/logstream.h>
+
+#include <deal.II/lac/full_matrix.h>
+#include <deal.II/lac/vector.h>
+
+#include <deal.II/optimization/line_minimization.h>
+#include <deal.II/optimization/solver_bfgs.h>
+
+#include <fstream>
+#include <iostream>
+
+#include "../tests.h"
+
+using namespace LineMinimization;
+
+template <typename number>
+void
+test()
+{
+  auto &out = deallog.get_file_stream();
+  out << std::setprecision(5) << std::fixed << std::right;
+
+  typedef Vector<number> VectorType;
+
+  // size of the problem
+  const unsigned int N = 21;
+
+  // parameters:
+  const unsigned int itmax = 150;
+  const double       gtol  = 1e-5; // gradient tolerance
+  const unsigned int m_max = 3;
+
+  // solution
+  VectorType x(N), x_shifted(x), x0(x);
+  x = 0.;
+
+  // shift minimizer to this point
+  VectorType location(x);
+  for (unsigned int i = 0; i < N; ++i)
+    location(i) = double(i) / (N - 1);
+
+  // see
+  // https://sourceforge.net/p/octave/optim/ci/default/tree/inst/rosenbrock.m#l26
+  const auto rosenbrok = [&](VectorType &x, VectorType &g) {
+    const unsigned int N   = x.size();
+    double             res = 0.;
+    g                      = 0;
+    for (unsigned int i = 0; i < N; ++i)
+      {
+        const double xi2 = x(i) * x(i);
+
+        if (i < N - 1)
+          {
+            res += 100. * dealii::Utilities::fixed_power<2>(x(i + 1) - xi2) +
+                   dealii::Utilities::fixed_power<2>(1. - x(i));
+
+            g(i) += -400. * x(i) * (x(i + 1) - xi2) - 2. * (1. - x(i));
+          }
+
+        if (i > 0)
+          g(i) += 200. * (x(i) - x(i - 1) * x(i - 1));
+      }
+    return res;
+  };
+
+
+
+  unsigned int tot_fun_calls = 0;
+  const auto   func          = [&](const VectorType &x, VectorType &g) {
+    tot_fun_calls++;
+    for (unsigned int i = 0; i < x.size(); ++i)
+      x_shifted(i) = x(i) - location(i) + 1.;
+
+    return rosenbrok(x_shifted, g);
+  };
+
+  const auto preconditioner = [](VectorType &                         g,
+                                 const FiniteSizeHistory<VectorType> &s,
+                                 const FiniteSizeHistory<VectorType> &y) {
+    if (s.size() > 0)
+      {
+        // default preconditioning using the oldest {s,y} pair, see
+        // lbfgs_recursion() in __bfgsmin.cc of "optim" Octave package.
+        const unsigned int i  = s.size() - 1;
+        const double       yy = y[i] * y[i];
+        const double       sy = s[i] * y[i];
+        Assert(yy > 0 && sy > 0, ExcInternalError());
+        g *= sy / yy;
+      }
+  };
+
+  SolverControl solver_control(itmax, gtol, false);
+  typename SolverBFGS<VectorType>::AdditionalData data(m_max, false);
+  SolverBFGS<VectorType>                          solver(solver_control, data);
+  solver.connect_preconditioner_slot(preconditioner);
+  solver.solve(func, x);
+
+  deallog << "Limited memory BFGS solution:" << std::endl;
+  x.print(deallog);
+
+  deallog << "Function value: " << func(x, x0) << std::endl;
+
+  x.add(-1, location);
+  deallog << "Linf error in solution: " << x.linfty_norm() << std::endl;
+
+  deallog << "function calls: "
+          << (tot_fun_calls - 1) /*one evaluation above*/ << std::endl;
+}
+
+int
+main()
+{
+  std::ofstream logfile("output");
+  deallog << std::setprecision(5);
+  deallog.attach(logfile);
+
+  test<double>();
+}

diff --git a/tests/optimization/bfgs_05b.output b/tests/optimization/bfgs_05b.output
new file mode 100644 (file)
index 0000000..f76da98
--- /dev/null
+++ b/tests/optimization/bfgs_05b.output
@@ -0,0 +1,8 @@
+
+DEAL:BFGS::Starting value 63.201
+DEAL:BFGS::Convergence step 127 value 8.1397e-06
+DEAL::Limited memory BFGS solution:
+DEAL::-4.4833e-09 0.050000 0.10000 0.15000 0.20000 0.25000 0.30000 0.35000 0.40000 0.45000 0.50000 0.55000 0.60000 0.65000 0.70000 0.75000 0.80000 0.85000 0.90000 0.95000 1.0000
+DEAL::Function value: 1.3096e-13
+DEAL::Linf error in solution: 1.6564e-07
+DEAL::function calls: 133