From e6261734e4db95e70b57b7d9bdc552d6ca755cf1 Mon Sep 17 00:00:00 2001 From: Matthias Maier Date: Mon, 30 Mar 2015 20:05:43 +0200 Subject: [PATCH] Testsuite: Cleanup: Put "run" and "diff" stage into separate script It was not a brilliant idea to begin with to exploit implementational (and unstable) features of CMake's generators for our test logic. And the breakage of the latest changes with the ninja generator evidence this fact. Therefore, phase them out into its own sh script and just call it for the test stage in mind. --- cmake/macros/macro_deal_ii_add_test.cmake | 81 ++++---------- cmake/macros/macro_deal_ii_pickup_tests.cmake | 39 +++---- cmake/scripts/CMakeLists.txt | 1 + cmake/scripts/run_test.sh | 103 ++++++++++++++++++ 4 files changed, 141 insertions(+), 83 deletions(-) create mode 100644 cmake/scripts/run_test.sh diff --git a/cmake/macros/macro_deal_ii_add_test.cmake b/cmake/macros/macro_deal_ii_add_test.cmake index 29c7ddbf5d..dcaf5408b9 100644 --- a/cmake/macros/macro_deal_ii_add_test.cmake +++ b/cmake/macros/macro_deal_ii_add_test.cmake @@ -77,15 +77,11 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file) # A "binary" in the output file indicates binary output. In this case we # have to switch to plain diff instead of (possibly) numdiff, which can # only work on plain text files. - # - # TODO: The indirection with ${${_test_diff_variable}} is necessary to - # avoid quoting issues with the command line :-/ - come up with a fix for - # that - # - SET(_test_diff_variable TEST_DIFF) IF(_file MATCHES "\\.binary\\.") - SET(_test_diff_variable DIFF_EXECUTABLE) + SET(_test_diff ${DIFF_EXECUTABLE}) + ELSE() + SET(_test_diff ${NUMDIFF_EXECUTABLE}) ENDIF() # @@ -146,14 +142,14 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file) SET(_diff_target ${_target}.diff) # diff target name SET(_test_full ${_category}/${_test_name}.${_build_lowercase}) # full test name SET(_test_directory ${CMAKE_CURRENT_BINARY_DIR}/${_target}) # directory to run the test in - SET(_run_command ${_target}) # the command to issue + SET(_run_command ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target}) # the command to issue ELSE() SET(_diff_target ${_test_name}.mpirun${_n_cpu}.${_build_lowercase}.diff) # diff target name SET(_test_full ${_category}/${_test_name}.mpirun=${_n_cpu}.${_build_lowercase}) # full test name SET(_test_directory ${CMAKE_CURRENT_BINARY_DIR}/${_target}/mpirun=${_n_cpu}) # directory to run the test in - SET(_run_command ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_n_cpu} ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target} ${MPIEXEC_POSTFLAGS}) # the command to issue + SET(_run_command "${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_n_cpu} ${MPIEXEC_PREFLAGS} ${CMAKE_CURRENT_BINARY_DIR}/${_target}/${_target} ${MPIEXEC_POSTFLAGS}") # the command to issue ENDIF() @@ -201,23 +197,9 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file) # ADD_CUSTOM_COMMAND(OUTPUT ${_test_directory}/output - COMMAND - rm -f ${_test_directory}/failing_output - COMMAND - rm -f ${_test_directory}/output - COMMAND - ${_run_command} > ${_test_directory}/stdout - || ((test -f ${_test_directory}/output || - cp ${_test_directory}/stdout ${_test_directory}/output) - && mv ${_test_directory}/output ${_test_directory}/failing_output - && echo "${_test_full}: BUILD successful." - && echo "${_test_full}: RUN failed. ------ Result: ${_test_directory}/failing_output" - && echo "${_test_full}: RUN failed. ------ Partial output:" - && cat ${_test_directory}/failing_output - && exit 1) - COMMAND - test -f ${_test_directory}/output || - cp ${_test_directory}/stdout ${_test_directory}/output + COMMAND sh ${DEAL_II_DIR}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh + run "${_test_full}" "${_run_command}" "${_test_diff}" + "${DIFF_EXECUTABLE}" "${_comparison_file}" COMMAND ${PERL_EXECUTABLE} -pi ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/normalize.pl ${_test_directory}/output @@ -226,51 +208,28 @@ MACRO(DEAL_II_ADD_TEST _category _test_name _comparison_file) DEPENDS ${_target} ${DEAL_II_PATH}/${DEAL_II_SHARE_RELDIR}/scripts/normalize.pl + VERBATIM ) + ADD_CUSTOM_COMMAND(OUTPUT ${_test_directory}/diff - COMMAND - rm -f ${_test_directory}/failing_diff - COMMAND - touch ${_test_directory}/diff - COMMAND - # run diff or numdiff (if available) to determine - # whether files are the same. if they are not, output - # the first few lines of the output of numdiff, followed - # by the results of regular diff since the latter is just - # more readable - ${${_test_diff_variable}} # see comment above about redirection - ${_comparison_file} - ${_test_directory}/output - > ${_test_directory}/diff - || (mv ${_test_directory}/diff - ${_test_directory}/failing_diff - && echo "${_test_full}: BUILD successful." - && echo "${_test_full}: RUN successful." - && echo "${_test_full}: DIFF failed. ------ Source: ${_comparison_file}" - && echo "${_test_full}: DIFF failed. ------ Result: ${_test_directory}/output" - && echo "Check ${_test_directory}/output ${_comparison_file}" - && echo "${_test_full}: DIFF failed. ------ Diff: ${_test_directory}/failing_diff" - && echo "${_test_full}: DIFF failed. ------ First 8 lines of numdiff/diff output:" - && cat ${_test_directory}/failing_diff | head -n 8 - && echo "${_test_full}: DIFF failed. ------ First 50 lines diff output:" - && ${DIFF_EXECUTABLE} -c ${_comparison_file} ${_test_directory}/output | head -n 50 - && exit 1) + COMMAND sh ${DEAL_II_DIR}/${DEAL_II_SHARE_RELDIR}/scripts/run_test.sh + diff "${_test_full}" "${_run_command}" "${_test_diff}" + "${DIFF_EXECUTABLE}" "${_comparison_file}" WORKING_DIRECTORY ${_test_directory} DEPENDS ${_test_directory}/output ${_comparison_file} + VERBATIM ) ADD_CUSTOM_TARGET(${_diff_target} - COMMAND - echo "${_test_full}: BUILD successful." - && echo "${_test_full}: RUN successful." - && echo "${_test_full}: DIFF successful." - && echo "${_test_full}: PASSED." - DEPENDS - ${_test_directory}/diff - ) + COMMAND echo "${_test_full}: BUILD successful." + COMMAND echo "${_test_full}: RUN successful." + COMMAND echo "${_test_full}: DIFF successful." + COMMAND echo "${_test_full}: PASSED." + DEPENDS ${_test_directory}/diff + ) # # And finally add the test: diff --git a/cmake/macros/macro_deal_ii_pickup_tests.cmake b/cmake/macros/macro_deal_ii_pickup_tests.cmake index 0d7d3c9717..2ecbe1da75 100644 --- a/cmake/macros/macro_deal_ii_pickup_tests.cmake +++ b/cmake/macros/macro_deal_ii_pickup_tests.cmake @@ -24,10 +24,6 @@ # and the following variables, that can be overwritten by environment or # command line: # -# TEST_DIFF -# - specifying the executable and command line of the diff command to -# use -# # TEST_LIBRARIES # TEST_LIBRARIES_DEBUG # TEST_LIBRARIES_RELEASE @@ -97,27 +93,26 @@ MACRO(DEAL_II_PICKUP_TESTS) MARK_AS_ADVANCED(DIFF_EXECUTABLE NUMDIFF_EXECUTABLE) - SET_IF_EMPTY(TEST_DIFF "$ENV{TEST_DIFF}") - IF("${TEST_DIFF}" STREQUAL "") - # - # No TEST_DIFF is set, specify one: - # + IF( NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND" AND + DIFF_EXECUTABLE MATCHES "-NOTFOUND" ) + MESSAGE(FATAL_ERROR + "Could not find diff or numdiff. One of those are required for running the testsuite.\n" + "Please specify DIFF_DIR or NUMDIFF_DIR to a location containing the binaries." + ) + ENDIF() - IF(NOT NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND") - SET(TEST_DIFF ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \\t\\n:<>=,;') - IF(DIFF_EXECUTABLE MATCHES "-NOTFOUND") - SET(DIFF_EXECUTABLE ${NUMDIFF_EXECUTABLE}) - ENDIF() - ELSEIF(NOT DIFF_EXECUTABLE MATCHES "-NOTFOUND") - SET(TEST_DIFF ${DIFF_EXECUTABLE}) - ELSE() - MESSAGE(FATAL_ERROR - "Could not find diff or numdiff. One of those are required for running the testsuite.\n" - "Please specify TEST_DIFF by hand." - ) - ENDIF() + IF(DIFF_EXECUTABLE MATCHES "-NOTFOUND") + SET(DIFF_EXECUTABLE ${NUMDIFF_EXECUTABLE}) + ENDIF() + + IF(NUMDIFF_EXECUTABLE MATCHES "-NOTFOUND") + SET(NUMDIFF_EXECUTABLE ${DIFF_EXECUTABLE}) ENDIF() + # + # Set time limit: + # + SET_IF_EMPTY(TEST_TIME_LIMIT "$ENV{TEST_TIME_LIMIT}") SET_IF_EMPTY(TEST_TIME_LIMIT 600) diff --git a/cmake/scripts/CMakeLists.txt b/cmake/scripts/CMakeLists.txt index 3feceadd95..7800ac97f0 100644 --- a/cmake/scripts/CMakeLists.txt +++ b/cmake/scripts/CMakeLists.txt @@ -22,6 +22,7 @@ SET_TARGET_PROPERTIES(expand_instantiations_exe PROPERTIES SET(_scripts ${CMAKE_CURRENT_SOURCE_DIR}/normalize.pl ${CMAKE_CURRENT_SOURCE_DIR}/run_test.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/run_test.sh ) FILE(COPY ${_scripts} DESTINATION ${CMAKE_BINARY_DIR}/${DEAL_II_SHARE_RELDIR}/scripts diff --git a/cmake/scripts/run_test.sh b/cmake/scripts/run_test.sh new file mode 100644 index 0000000000..af073b6809 --- /dev/null +++ b/cmake/scripts/run_test.sh @@ -0,0 +1,103 @@ +## --------------------------------------------------------------------- +## +## Copyright (C) 2015 by the deal.II authors +## +## This file is part of the deal.II library. +## +## The deal.II library is free software; you can use it, redistribute +## it, and/or modify it under the terms of the GNU Lesser General +## Public License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## The full text of the license can be found in the file LICENSE at +## the top level of the deal.II distribution. +## +## --------------------------------------------------------------------- + +# +# Helper script used in testsuite targets to run ("run") and compare +# ("diff") individual tests. +# +# Usage: +# run_test.sh run TEST_FULL RUN_COMMAND +# +# run_test.sh diff TEST_FULL DIFF_EXECUTABLE DIFF_EXECUTABLE COMPARISON_FILE +# + +set -u + +STAGE="$1" +TEST_FULL="$2" +RUN_COMMAND="$3" +NUMDIFF_EXECUTABLE="$5" +DIFF_EXECUTABLE="$5" +COMPARISON_FILE="$6" + +# +# Add a top level target to run and compare the test: +# + +run(){ + rm -f failing_output + rm -f output + + ${RUN_COMMAND} > stdout + RETURN_VALUE=$? + + [ -f output ] || cp stdout output + rm -f stdout + + if [ $RETURN_VALUE -ne 0 ]; then + mv output failing_output + echo "${TEST_FULL}: BUILD successful." + echo "${TEST_FULL}: RUN failed. ------ Result: `pwd`/failing_output" + echo "${TEST_FULL}: RUN failed. ------ Partial output:" + cat failing_output + exit 1 + fi +} + +diff() { + rm -f failing_diff + touch diff + + # + # run diff or numdiff (if available) to determine whether files are the + # same. if they are not, output the first few lines of the output of + # numdiff, followed by the results of regular diff since the latter is + # just more readable + # + + case ${NUMDIFF_EXECUTABLE} in + *numdiff) + ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \t\n:<>=,;' \ + "${COMPARISON_FILE}" output > diff + ;; + *) + "${DIFF_EXECUTABLE}" "${COMPARISON_FILE}" output > diff + esac + + if [ $? -ne 0 ]; then + mv diff /failing_diff + echo "${TEST_FULL}: BUILD successful." + echo "${TEST_FULL}: RUN successful." + echo "${TEST_FULL}: DIFF failed. ------ Source: ${COMPARISON_FILE}" + echo "${TEST_FULL}: DIFF failed. ------ Result: `pwd`/output" + echo "Check `pwd`/output ${COMPARISON_FILE}" + echo "${TEST_FULL}: DIFF failed. ------ Diff: `pwd`/failing_diff" + echo "${TEST_FULL}: DIFF failed. ------ First 8 lines of numdiff/diff output:" + cat failing_diff | head -n 8 + echo "${TEST_FULL}: DIFF failed. ------ First 50 lines diff output:" + "${DIFF_EXECUTABLE}" -c "${COMPARISON_FILE}" output | head -n 50 + exit 1 + fi + exit 0 +} + +case $STAGE in + run) + run;; + diff) + diff;; + *) + exit 1;; +esac -- 2.39.5