From 63330763b3650933b408b535454324d6200c46f9 Mon Sep 17 00:00:00 2001
From: Matthias Maier <tamiko@43-1.org>
Date: Wed, 9 Feb 2022 16:57:42 -0600
Subject: [PATCH] Performance tests: add ctest wrapper

---
 tests/performance/collect_measurements | 51 ++++++++++++++++++++++++++
 tests/run_performance_tests.cmake      | 12 ++++++
 tests/run_testsuite.cmake              | 29 +++++++++++----
 3 files changed, 85 insertions(+), 7 deletions(-)
 create mode 100755 tests/performance/collect_measurements
 create mode 100644 tests/run_performance_tests.cmake

diff --git a/tests/performance/collect_measurements b/tests/performance/collect_measurements
new file mode 100755
index 0000000000..aecaec5d41
--- /dev/null
+++ b/tests/performance/collect_measurements
@@ -0,0 +1,51 @@
+#!/bin/bash
+## ---------------------------------------------------------------------
+##
+## Copyright (C) 2022 by the deal.II authors
+##
+## This file is part of the deal.II library.
+##
+## The deal.II library is free software; you can use it, redistribute
+## it, and/or modify it under the terms of the GNU Lesser General
+## Public License as published by the Free Software Foundation; either
+## version 2.1 of the License, or (at your option) any later version.
+## The full text of the license can be found in the file LICENSE.md at
+## the top level directory of deal.II.
+##
+## ---------------------------------------------------------------------
+
+#
+# Performance tests record timing/sensor statistics in the output file
+# "output".
+#
+# This script collects all individual measurements from these files and
+# pretty prints a parsable table to stdout.
+#
+
+for i in tests/performance/*/output tests/performance/*/mpirun*/output
+do
+  # Skip output files that have fewer than 3 lines
+  if [[ "$(wc -l $i 2> /dev/null | awk '{ print $1 }')" -le 2 ]]; then
+    continue
+  fi
+
+  # Recover test name from full file path stored in ${i}
+  test_name=${i##tests/performance/}
+  test_name=${test_name%%/output}
+  test_name=${test_name/.release/}
+  test_name=${test_name/.debug/}
+
+  {
+    read -r test_type # first line of output file contains test type
+    read -r           # skip second line
+
+    # iterate over all remaining lines and output a table:
+    while read -r line
+    do
+      echo -e "$test_name\t$test_type\t${line}"
+    done
+  } < "${i}"
+done | sort |
+  column -s $'\t' -t \
+    -N "test_name,test_type,sensor,min/value,max,mean,std_dev,samples"
+
diff --git a/tests/run_performance_tests.cmake b/tests/run_performance_tests.cmake
new file mode 100644
index 0000000000..b7dfc57c39
--- /dev/null
+++ b/tests/run_performance_tests.cmake
@@ -0,0 +1,12 @@
+SET(TRACK "Experimental")
+
+SET(CMAKE_BUILD_TYPE Release)
+SET(DEAL_II_COMPILE_EXAMPLES FALSE)
+SET(TEST_PICKUP_REGEX "^performance")
+
+SET(ENABLE_PERFORMANCE_TESTS TRUE)
+IF(NOT DEFINED SKIP_SUBMISSION)
+  SET(SKIP_SUBMISSION TRUE)
+ENDIF()
+
+INCLUDE(${CMAKE_CURRENT_LIST_DIR}/run_testsuite.cmake)
diff --git a/tests/run_testsuite.cmake b/tests/run_testsuite.cmake
index 39f6c57c03..098da5e037 100644
--- a/tests/run_testsuite.cmake
+++ b/tests/run_testsuite.cmake
@@ -83,7 +83,7 @@
 #
 #   MEMORYCHECK
 #     - If set to ON the CTEST_MEMORYCHECK() stage will be run.
-#     Test results must go into the "Experimantal" section.
+#     Test results must go into the "Experimental" section.
 #
 #   MAKEOPTS
 #     - Additional options that will be passed directly to make (or ninja).
@@ -270,7 +270,7 @@ ENDIF()
 # Pass all relevant variables down to configure:
 GET_CMAKE_PROPERTY(_variables VARIABLES)
 FOREACH(_var ${_variables})
-  IF( _var MATCHES "^(TEST|DEAL_II|ALLOW|WITH|FORCE|COMPONENT)_" OR
+  IF( _var MATCHES "^(ENABLE|TEST|DEAL_II|ALLOW|WITH|FORCE|COMPONENT)_" OR
       _var MATCHES "^(DOCUMENTATION|EXAMPLES)" OR
       _var MATCHES "^(ADOLC|ARBORX|ARPACK|BOOST|OPENCASCADE|MUPARSER|HDF5|KOKKOS|METIS|MPI)_" OR
       _var MATCHES "^(GINKGO|P4EST|PETSC|SCALAPACK|SLEPC|THREADS|TBB|TRILINOS)_" OR
@@ -577,15 +577,30 @@ Unable to determine test submission files from TAG. Bailing out.
     )
 ENDIF()
 
+#
+# Create performance test report
+#
+IF(ENABLE_PERFORMANCE_TESTS)
+  MESSAGE("-- Collecting performance measurements")
+  EXECUTE_PROCESS(
+    COMMAND bash "${CMAKE_CURRENT_LIST_DIR}/performance/collect_measurements"
+    WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
+    OUTPUT_FILE ${CTEST_BINARY_DIRECTORY}/performance.log
+    )
+ENDIF()
+
+
 #
 # And finally submit:
 #
 
-MESSAGE("-- Running CTEST_SUBMIT()")
-CTEST_SUBMIT(RETURN_VALUE _res)
+IF(NOT ${SKIP_SUBMISSION})
+  MESSAGE("-- Running CTEST_SUBMIT()")
+  CTEST_SUBMIT(RETURN_VALUE _res)
 
-IF("${_res}" STREQUAL "0")
-  MESSAGE("-- Submission successful. Goodbye!")
+  IF("${_res}" STREQUAL "0")
+    MESSAGE("-- Submission successful. Goodbye!")
+  ENDIF()
 ENDIF()
 
-# .oO( This script is freaky 600 lines long... )
+# .oO( This script is freaky 606 lines long... )
-- 
2.39.5