From: Matthias Maier Date: Sun, 30 Aug 2015 01:30:08 +0000 (-0500) Subject: Testsuite: Allow comparison against output variants X-Git-Tag: v8.4.0-rc2~519^2~8 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=da72eaf8d3d11c014cfe7db5a365b89d4dbc2c9b;p=dealii.git Testsuite: Allow comparison against output variants This commit modifies run_test.sh such that it also considers all variants of the form [...].output [...].output.[STRING] for comparison. If a numdiff/diff against one of those variants succeeds a test is considered succesful. --- diff --git a/cmake/scripts/run_test.sh b/cmake/scripts/run_test.sh index 9a2ff609ff..f0df2eb008 100644 --- a/cmake/scripts/run_test.sh +++ b/cmake/scripts/run_test.sh @@ -32,6 +32,9 @@ NUMDIFF_EXECUTABLE="$4" DIFF_EXECUTABLE="$5" COMPARISON_FILE="$6" +# Ensure uniform sorting for pathname expansion +export LC_ALL=C + # # Add a top level target to run and compare the test: # @@ -64,27 +67,65 @@ run(){ } diff() { - rm -f failing_diff + rm -f failing_diff* + rm -f diff* touch diff + test_successful=false + # - # run diff or numdiff (if available) to determine whether files are the - # same. if they are not, output the first few lines of the output of - # numdiff, followed by the results of regular diff since the latter is - # just more readable + # Pick up main comparison file and all variants. A valid variant name is + # of the form [...].output.[STRING] # + for file in "${COMPARISON_FILE}" "${COMPARISON_FILE}".*; do + # determine variant name (empty string for main comparison file): + variant=${file#*.output} + + # + # Run diff or numdiff (if available) to determine whether files are the + # same. Create a diff file "diff${variant}" for each variant file that + # is found (including the main comparison file). + # + case ${NUMDIFF_EXECUTABLE} in + *numdiff) + ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \t\n:<>=,;' \ + "${file}" output > diff${variant} + ;; + *) + "${DIFF_EXECUTABLE}" "${file}" output > diff${variant} + ;; + esac + + if [ $? -eq 0 ]; then + # + # Ensure that only a single diff file with no contents remains + # (numdiff has the bad habit of being very verbose...): + # + rm -f diff* + touch diff + + if [ -n "${variant}" ]; then + # + # In case of a successful comparison against a variant, store the + # fact that we compared against a variant in the diff file. + # + echo "${TEST_FULL}: DIFF successful. - Variant: ${file}" > diff + fi - case ${NUMDIFF_EXECUTABLE} in - *numdiff) - ${NUMDIFF_EXECUTABLE} -a 1e-6 -r 1e-8 -s ' \t\n:<>=,;' \ - "${COMPARISON_FILE}" output > diff - ;; - *) - "${DIFF_EXECUTABLE}" "${COMPARISON_FILE}" output > diff - esac - - if [ $? -ne 0 ]; then - mv diff failing_diff + test_successful=true + break + fi + done + + # + # If none of the diffs succeeded, use the diff against the main comparison + # file. Output the first few lines of the output of numdiff, followed by + # the results of regular diff since the latter is just more readable. + # + if [ $test_successful = false ] ; then + for file in diff*; do + mv "$file" failing_"$file" + done echo "${TEST_FULL}: BUILD successful." echo "${TEST_FULL}: RUN successful." echo "${TEST_FULL}: DIFF failed. ------ Source: ${COMPARISON_FILE}"