]> https://gitweb.dealii.org/ - dealii.git/commitdiff
Testsuite: Cleanup: Put "run" and "diff" stage into separate script
authorMatthias Maier <tamiko@43-1.org>
Mon, 30 Mar 2015 18:05:43 +0000 (20:05 +0200)
committerMatthias Maier <tamiko@43-1.org>
Mon, 30 Mar 2015 20:40:06 +0000 (22:40 +0200)
It was not a brilliant idea to begin with to exploit implementational (and
unstable) features of CMake's generators for our test logic. And the
breakage of the latest changes with the ninja generator evidence this fact.

Therefore, phase them out into its own sh script and just call it for the
test stage in mind.

cmake/macros/macro_deal_ii_add_test.cmake
cmake/macros/macro_deal_ii_pickup_tests.cmake
cmake/scripts/CMakeLists.txt
cmake/scripts/run_test.sh [new file with mode: 0644]

index 29c7ddbf5d24088d662a147cd98633e721d1d0af..dcaf5408b94fd38fd6af091df92c1e9294502236 100644 (file)
@@ -77,15 +77,11 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
   # A "binary" in the output file indicates binary output. In this case we
   # have to switch to plain diff instead of (possibly) numdiff, which can
   # only work on plain text files.
-  #
-  # TODO: The indirection with ${${_test_diff_variable}} is necessary to
-  # avoid quoting issues with the command line :-/ - come up with a fix for
-  # that
-  #
 
-  SET(_test_diff_variable TEST_DIFF)
   IF(_file MATCHES "\\.binary\\.")
-    SET(_test_diff_variable DIFF_EXECUTABLE)
+    SET(_test_diff ${DIFF_EXECUTABLE})
+  ELSE()
+    SET(_test_diff ${NUMDIFF_EXECUTABLE})
   ENDIF()
 
   #
@@ -146,14 +142,14 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
         SET(_diff_target ${_target}.diff) # diff target name
         SET(_test_full ${_category}/${_test_name}.${_build_lowercase}) # full test name
         SET(_test_directory ${CMAKE_CURRENT_BINARY_DIR}/${_target}) # directory to run the test in
-        SET(_run_command ${_target}) # the command to issue
+        SET(_run_command ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target}) # the command to issue
 
       ELSE()
 
         SET(_diff_target ${_test_name}.mpirun${_n_cpu}.${_build_lowercase}.diff) # diff target name
         SET(_test_full ${_category}/${_test_name}.mpirun=${_n_cpu}.${_build_lowercase}) # full test name
         SET(_test_directory ${CMAKE_CURRENT_BINARY_DIR}/${_target}/mpirun=${_n_cpu}) # directory to run the test in
-        SET(_run_command ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_n_cpu} ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target} ${MPIEXEC_POSTFLAGS}) # the command to issue
+        SET(_run_command "${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_n_cpu} ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target} ${MPIEXEC_POSTFLAGS}") # the command to issue
 
       ENDIF()
 
@@ -201,23 +197,9 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
       #
 
       ADD_CUSTOM_COMMAND(OUTPUT ${_test_directory}/output
-        COMMAND
-          rm -f ${_test_directory}/failing_output
-        COMMAND
-          rm -f ${_test_directory}/output
-        COMMAND
-          ${_run_command} > ${_test_directory}/stdout
-          || ((test -f ${_test_directory}/output ||
-               cp ${_test_directory}/stdout ${_test_directory}/output)
-              && mv ${_test_directory}/output ${_test_directory}/failing_output
-              && echo "${_test_full}: BUILD successful."
-              && echo "${_test_full}: RUN failed. ------ Result: ${_test_directory}/failing_output"
-              && echo "${_test_full}: RUN failed. ------ Partial output:"
-              && cat ${_test_directory}/failing_output
-              && exit 1)
-        COMMAND
-          test -f ${_test_directory}/output ||
-          cp ${_test_directory}/stdout ${_test_directory}/output
+        COMMAND sh ${DEAL_II_DIR}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh
+          run "${_test_full}" "${_run_command}" "${_test_diff}"
+          "${DIFF_EXECUTABLE}" "${_comparison_file}"
         COMMAND ${PERL_EXECUTABLE}
           -pi ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/normalize.pl
           ${_test_directory}/output
@@ -226,51 +208,28 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file)
         DEPENDS
           ${_target}
           ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/normalize.pl
+        VERBATIM
         )
+
       ADD_CUSTOM_COMMAND(OUTPUT ${_test_directory}/diff
-        COMMAND
-          rm -f ${_test_directory}/failing_diff
-        COMMAND
-          touch ${_test_directory}/diff
-        COMMAND
-          # run diff or numdiff (if available) to determine
-          # whether files are the same. if they are not, output
-          # the first few lines of the output of numdiff, followed
-          # by the results of regular diff since the latter is just
-          # more readable
-          ${${_test_diff_variable}} # see comment above about redirection
-            ${_comparison_file}
-            ${_test_directory}/output
-            > ${_test_directory}/diff
-          || (mv ${_test_directory}/diff
-                 ${_test_directory}/failing_diff
-              && echo "${_test_full}: BUILD successful."
-              && echo "${_test_full}: RUN successful."
-              && echo "${_test_full}: DIFF failed. ------ Source: ${_comparison_file}"
-              && echo "${_test_full}: DIFF failed. ------ Result: ${_test_directory}/output"
-              && echo "Check ${_test_directory}/output ${_comparison_file}"
-              && echo "${_test_full}: DIFF failed. ------ Diff:   ${_test_directory}/failing_diff"
-              && echo "${_test_full}: DIFF failed. ------ First 8 lines of numdiff/diff output:"
-              && cat ${_test_directory}/failing_diff | head -n 8
-              && echo "${_test_full}: DIFF failed. ------ First 50 lines diff output:"
-              && ${DIFF_EXECUTABLE} -c ${_comparison_file} ${_test_directory}/output | head -n 50
-              && exit 1)
+        COMMAND sh ${DEAL_II_DIR}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh
+          diff "${_test_full}" "${_run_command}" "${_test_diff}"
+          "${DIFF_EXECUTABLE}" "${_comparison_file}"
         WORKING_DIRECTORY
           ${_test_directory}
         DEPENDS
           ${_test_directory}/output
           ${_comparison_file}
+        VERBATIM
         )
 
       ADD_CUSTOM_TARGET(${_diff_target}
-        COMMAND
-             echo "${_test_full}: BUILD successful."
-          && echo "${_test_full}: RUN successful."
-          && echo "${_test_full}: DIFF successful."
-          && echo "${_test_full}: PASSED."
-        DEPENDS
-          ${_test_directory}/diff
-      )
+        COMMAND echo "${_test_full}: BUILD successful."
+        COMMAND echo "${_test_full}: RUN successful."
+        COMMAND echo "${_test_full}: DIFF successful."
+        COMMAND echo "${_test_full}: PASSED."
+        DEPENDS ${_test_directory}/diff
+        )
 
       #
       # And finally add the test:
index 0d7d3c9717410ffa8772c05440af5978075e3097..2ecbe1da758260c31b681ecb9d51f3679adbbe63 100644 (file)
 # and the following variables, that can be overwritten by environment or
 # command line:
 #
-#     TEST_DIFF
-#       - specifying the executable and command line of the diff command to
-#         use
-#
 #     TEST_LIBRARIES
 #     TEST_LIBRARIES_DEBUG
 #     TEST_LIBRARIES_RELEASE
@@ -97,27 +93,26 @@ MACRO(DEAL_II_PICKUP_TESTS)
 
   MARK_AS_ADVANCED(DIFF_EXECUTABLE NUMDIFF_EXECUTABLE)
 
-  SET_IF_EMPTY(TEST_DIFF "$ENV{TEST_DIFF}")
-  IF("${TEST_DIFF}" STREQUAL "")
-    #
-    # No TEST_DIFF is set, specify one:
-    #
+  IF( NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND" AND
+      DIFF_EXECUTABLE MATCHES "-NOTFOUND" )
+    MESSAGE(FATAL_ERROR
+      "Could not find diff or numdiff. One of those are required for running the testsuite.\n"
+      "Please specify DIFF_DIR or NUMDIFF_DIR to a location containing the binaries."
+      )
+  ENDIF()
 
-    IF(NOT NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND")
-      SET(TEST_DIFF ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \\t\\n:<>=,;')
-      IF(DIFF_EXECUTABLE MATCHES "-NOTFOUND")
-        SET(DIFF_EXECUTABLE ${NUMDIFF_EXECUTABLE})
-      ENDIF()
-    ELSEIF(NOT DIFF_EXECUTABLE MATCHES "-NOTFOUND")
-      SET(TEST_DIFF ${DIFF_EXECUTABLE})
-    ELSE()
-      MESSAGE(FATAL_ERROR
-        "Could not find diff or numdiff. One of those are required for running the testsuite.\n"
-        "Please specify TEST_DIFF by hand."
-        )
-    ENDIF()
+  IF(DIFF_EXECUTABLE MATCHES "-NOTFOUND")
+    SET(DIFF_EXECUTABLE ${NUMDIFF_EXECUTABLE})
+  ENDIF()
+
+  IF(NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND")
+    SET(NUMDIFF_EXECUTABLE ${DIFF_EXECUTABLE})
   ENDIF()
 
+  #
+  # Set time limit:
+  #
+
   SET_IF_EMPTY(TEST_TIME_LIMIT "$ENV{TEST_TIME_LIMIT}")
   SET_IF_EMPTY(TEST_TIME_LIMIT 600)
 
index 3feceadd9578e74a0be2c657279cb29c0f792759..7800ac97f0ef71a9619cb28bcbc167185e63868c 100644 (file)
@@ -22,6 +22,7 @@ SET_TARGET_PROPERTIES(expand_instantiations_exe PROPERTIES
 SET(_scripts
   ${CMAKE_CURRENT_SOURCE_DIR}/normalize.pl
   ${CMAKE_CURRENT_SOURCE_DIR}/run_test.cmake
+  ${CMAKE_CURRENT_SOURCE_DIR}/run_test.sh
   )
 FILE(COPY ${_scripts}
   DESTINATION ${CMAKE_BINARY_DIR}/${DEAL_II_SHARE_RELDIR}/scripts
diff --git a/cmake/scripts/run_test.sh b/cmake/scripts/run_test.sh
new file mode 100644 (file)
index 0000000..af073b6
--- /dev/null
@@ -0,0 +1,103 @@
+## ---------------------------------------------------------------------
+##
+## Copyright (C) 2015 by the deal.II authors
+##
+## This file is part of the deal.II library.
+##
+## The deal.II library is free software; you can use it, redistribute
+## it, and/or modify it under the terms of the GNU Lesser General
+## Public License as published by the Free Software Foundation; either
+## version 2.1 of the License, or (at your option) any later version.
+## The full text of the license can be found in the file LICENSE at
+## the top level of the deal.II distribution.
+##
+## ---------------------------------------------------------------------
+
+#
+# Helper script used in testsuite targets to run ("run") and compare
+# ("diff") individual tests.
+#
+# Usage:
+#   run_test.sh run TEST_FULL RUN_COMMAND
+#
+#   run_test.sh diff TEST_FULL DIFF_EXECUTABLE DIFF_EXECUTABLE COMPARISON_FILE
+#
+
+set -u
+
+STAGE="$1"
+TEST_FULL="$2"
+RUN_COMMAND="$3"
+NUMDIFF_EXECUTABLE="$5"
+DIFF_EXECUTABLE="$5"
+COMPARISON_FILE="$6"
+
+#
+# Add a top level target to run and compare the test:
+#
+
+run(){
+  rm -f failing_output
+  rm -f output
+
+  ${RUN_COMMAND} > stdout
+  RETURN_VALUE=$?
+
+  [ -f output ] || cp stdout output
+  rm -f stdout
+
+  if [ $RETURN_VALUE -ne 0 ]; then
+    mv output failing_output
+    echo "${TEST_FULL}: BUILD successful."
+    echo "${TEST_FULL}: RUN failed. ------ Result: `pwd`/failing_output"
+    echo "${TEST_FULL}: RUN failed. ------ Partial output:"
+    cat failing_output
+    exit 1
+  fi
+}
+
+diff() {
+  rm -f failing_diff
+  touch diff
+
+  #
+  # run diff or numdiff (if available) to determine whether files are the
+  # same. if they are not, output the first few lines of the output of
+  # numdiff, followed by the results of regular diff since the latter is
+  # just more readable
+  #
+
+  case ${NUMDIFF_EXECUTABLE} in
+    *numdiff)
+      ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \t\n:<>=,;' \
+                            "${COMPARISON_FILE}" output > diff
+      ;;
+    *)
+      "${DIFF_EXECUTABLE}" "${COMPARISON_FILE}" output > diff
+  esac
+
+  if [ $? -ne 0 ]; then
+    mv diff /failing_diff
+    echo "${TEST_FULL}: BUILD successful."
+    echo "${TEST_FULL}: RUN successful."
+    echo "${TEST_FULL}: DIFF failed. ------ Source: ${COMPARISON_FILE}"
+    echo "${TEST_FULL}: DIFF failed. ------ Result: `pwd`/output"
+    echo "Check `pwd`/output ${COMPARISON_FILE}"
+    echo "${TEST_FULL}: DIFF failed. ------ Diff:   `pwd`/failing_diff"
+    echo "${TEST_FULL}: DIFF failed. ------ First 8 lines of numdiff/diff output:"
+    cat failing_diff | head -n 8
+    echo "${TEST_FULL}: DIFF failed. ------ First 50 lines diff output:"
+    "${DIFF_EXECUTABLE}" -c "${COMPARISON_FILE}" output | head -n 50
+    exit 1
+  fi
+  exit 0
+}
+
+case $STAGE in
+  run)
+    run;;
+  diff)
+    diff;;
+  *)
+    exit 1;;
+esac

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.