boxes/test/testrunner.sh

301 lines
8.6 KiB
Bash
Raw Normal View History

2014-09-23 23:50:36 +02:00
#!/usr/bin/env bash
#
# boxes - Command line filter to draw/remove ASCII boxes around text
2023-03-26 21:32:08 +02:00
# Copyright (c) 1999-2023 Thomas Jensen and the boxes contributors
#
2022-09-18 14:56:30 +02:00
# This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public
# License, version 3, as published by the Free Software Foundation.
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
2022-09-18 14:56:30 +02:00
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
# You should have received a copy of the GNU General Public License along with this program.
# If not, see <https://www.gnu.org/licenses/>.
#____________________________________________________________________________________________________________________
2014-09-23 23:50:36 +02:00
#
# Test runner for the black-box tests.
2022-09-18 14:56:30 +02:00
#____________________________________________________________________________________________________________________
2014-09-23 23:50:36 +02:00
set -uo pipefail
# Global constants
declare -r SRC_DIR=../src
declare -r OUT_DIR=../out
declare -r BASELINE_FILE=${OUT_DIR}/lcov-baseline.info
2021-10-20 22:28:34 +02:00
declare -r COVERAGE_FILE=${OUT_DIR}/lcov-total.info
# Command Line Options
declare opt_coverage=false
declare opt_suite=false
declare opt_testCase=""
function print_usage()
{
echo 'Usage: testrunner.sh [--coverage] {--suite | <opt_testCase>}'
2014-09-23 23:50:36 +02:00
echo ' Returns 0 for success, else non-zero'
}
function parse_arguments()
{
if [ $# -eq 0 ]; then
print_usage
exit 2
fi
for i in "$@"; do
case ${i} in
--coverage)
opt_coverage=true
shift
;;
--suite)
opt_suite=true
shift
;;
-h | --help)
print_usage
exit 0
;;
*)
if [ -z "${opt_testCase}" ]; then
opt_testCase=${i}
else
print_usage
exit 2
fi
;;
esac
done
if [[ ${opt_testCase} == "" && ${opt_suite} != "true" ]]; then
print_usage
exit 2
fi
}
function check_prereqs()
{
if [ "${PWD##*/}" != "test" ]; then
>&2 echo "Please run this script from the test folder."
exit 2
fi
if [ ! -d ${OUT_DIR} ]; then
>&2 echo "Please run 'make' from the project root to build an executable before running tests."
exit 2
fi
if [[ ${opt_coverage} == true && $(find ${OUT_DIR} -maxdepth 1 -name '*.gcno' 2>/dev/null | wc -l) -lt 1 ]]; then
>&2 echo "Binaries not instrumented. Run 'make cov' from the project root."
exit 5
fi
if [ ! -f ${opt_testCase} ]; then
>&2 echo "Test Case '${opt_testCase}' not found."
exit 3
fi
}
function cov_baseline()
{
if [ ${opt_coverage} == true ]; then
if [ ${opt_suite} == true ]; then
rm -f ${BASELINE_FILE}
fi
if [ ! -f ${BASELINE_FILE} ]; then
echo "Creating coverage baseline ..."
lcov --capture --initial --no-recursion --directory ${OUT_DIR} --base-directory ${SRC_DIR} \
--exclude '*/lex.yy.c' --exclude '*/parser.c' --output-file ${BASELINE_FILE}
echo -e "Coverage baseline created in ${BASELINE_FILE}\n"
fi
fi
}
2014-09-23 23:50:36 +02:00
function execute_suite()
{
local countExecuted=0
local countFailed=0
local tc
2014-09-23 23:50:36 +02:00
for tc in *.txt; do
if [ ${opt_coverage} == true ]; then
$0 --coverage "${tc}"
else
$0 "${tc}"
fi
2014-09-23 23:50:36 +02:00
if [ $? -ne 0 ]; then
overallResult=1
countFailed=$((countFailed + 1))
2014-09-23 23:50:36 +02:00
fi
countExecuted=$((countExecuted + 1))
2014-09-23 23:50:36 +02:00
done
echo "${countExecuted} tests executed, $((countExecuted - countFailed)) successful, ${countFailed} failed."
}
2014-09-23 23:50:36 +02:00
function measure_coverage()
{
local testResultsDir=${OUT_DIR}/test-results/${tcBaseName}
if [ ${opt_coverage} == true ]; then
mkdir -p "${testResultsDir}"
cp ${OUT_DIR}/*.gc* "${testResultsDir}"
lcov --capture --directory "${testResultsDir}" --base-directory ${SRC_DIR} --test-name "${tcBaseName}" --quiet \
--exclude '*/lex.yy.c' --exclude '*/parser.c' --rc branch_coverage=1 \
--output-file "${testResultsDir}/coverage.info"
echo -n " Coverage: "
lcov --summary "${testResultsDir}/coverage.info" 2>&1 | grep 'lines...' | grep -oP '\d+\.\d*%'
fi
}
2014-09-23 23:50:36 +02:00
function consolidate_coverage()
{
echo -e "\nConsolidating test coverage ..."
pushd ${OUT_DIR}/test-results || exit 1
find . -name "*.info" | xargs printf -- '--add-tracefile %s\n' | xargs --exit \
lcov --rc lcov_branch_coverage=1 --exclude '*/lex.yy.c' --exclude '*/parser.c' \
2023-11-25 20:35:30 +01:00
--ignore-errors unused --output-file ../${COVERAGE_FILE} --add-tracefile ../${BASELINE_FILE}
popd || exit 1
echo ""
}
function report_coverage()
{
2021-10-20 22:28:34 +02:00
local testReportDir=${OUT_DIR}/report
mkdir -p ${testReportDir}
2021-10-20 22:28:34 +02:00
genhtml --title "Boxes / All Tests" --branch-coverage --legend \
--output-directory ${testReportDir} ${COVERAGE_FILE}
echo -e "\nTest coverage report available at ${testReportDir}/index.html"
}
function clear_gcda_traces()
{
if [ ${opt_coverage} == true ]; then
rm -f ${OUT_DIR}/*.gcda
2014-09-23 23:50:36 +02:00
fi
}
2014-09-23 23:50:36 +02:00
function check_mandatory_sections()
{
local sectionName
for sectionName in :ARGS :INPUT :OUTPUT-FILTER :EXPECTED :EOF; do
if [ $(grep -c ^$sectionName $opt_testCase) -ne 1 ]; then
>&2 echo "Missing section $sectionName in test case '$opt_testCase'."
exit 4
fi
done
}
2014-09-23 23:50:36 +02:00
function arrange_environment()
{
local boxesEnv=""
if [ $(grep -c "^:ENV" ${opt_testCase}) -eq 1 ]; then
boxesEnv=$(cat ${opt_testCase} | sed -n '/^:ENV/,/^:ARGS/p;' | sed '1d;$d' | tr -d '\r')
fi
if [ -n "$boxesEnv" ]; then
echo "$boxesEnv" | sed -e 's/export/\n export/g' | sed '1d'
unset BOXES
eval "$boxesEnv"
else
export BOXES=../boxes-config
fi
}
2014-09-23 23:50:36 +02:00
function arrange_test_fixtures()
{
if [ $(grep -c "^:EXPECTED-ERROR " ${opt_testCase}) -eq 1 ]; then
expectedReturnCode=$(grep "^:EXPECTED-ERROR " ${opt_testCase} | sed -e 's/:EXPECTED-ERROR //')
fi
2014-09-23 23:50:36 +02:00
cat ${opt_testCase} | sed -n '/^:INPUT/,/^:OUTPUT-FILTER/p;' | sed '1d;$d' | tr -d '\r' > "${testInputFile}"
cat ${opt_testCase} | sed -n '/^:OUTPUT-FILTER/,/^:EXPECTED\b.*$/p;' | sed '1d;$d' | tr -d '\r' > "${testFilterFile}"
cat ${opt_testCase} | sed -n '/^:EXPECTED/,/^:EOF/p;' | sed '1d;$d' | tr -d '\r' > "${testExpectationFile}"
}
function run_boxes()
{
local boxesBinary=${OUT_DIR}/boxes.exe
if [ ! -x $boxesBinary ]; then
boxesBinary=${OUT_DIR}/boxes
fi
echo " Invoking: $(basename $boxesBinary) $boxesArgs"
if [ -z "${BOXES_TEST_XXD:-}" ]; then
eval "$boxesBinary $boxesArgs" < "$testInputFile" > "$testOutputFile" 2>&1
else
eval "$boxesBinary $boxesArgs" < "$testInputFile" | xxd > "$testOutputFile" 2>&1
fi
actualReturnCode=$?
}
function assert_outcome()
{
tr -d '\r' < "${testOutputFile}" | sed -E -f "${testFilterFile}" | diff - "${testExpectationFile}"
if [ $? -ne 0 ]; then
>&2 echo "Error in test case: ${opt_testCase} (top: actual; bottom: expected)"
exit 5
fi
if [ ${actualReturnCode} -ne "${expectedReturnCode}" ]; then
>&2 echo -n "Error in test case: ${opt_testCase}"
>&2 echo " (error code was ${actualReturnCode}, but expected ${expectedReturnCode})"
exit 5
fi
}
parse_arguments "$@"
check_prereqs
cov_baseline
# Execute the entire test suite
if [ ${opt_suite} == true ]; then
declare -i overallResult=0
execute_suite
if [ ${opt_coverage} == true ]; then
consolidate_coverage
report_coverage
fi
exit ${overallResult}
2014-09-23 23:50:36 +02:00
fi
# Execute only a single test
echo "Running test case: ${opt_testCase}"
declare -r tcBaseName=${opt_testCase%.txt}
clear_gcda_traces
check_mandatory_sections
declare -i expectedReturnCode=0
declare -r testInputFile=${opt_testCase/%.txt/.input.tmp}
declare -r testExpectationFile=${opt_testCase/%.txt/.expected.tmp}
declare -r testFilterFile=${opt_testCase/%.txt/.sed.tmp}
declare -r testOutputFile=${opt_testCase/%.txt/.out.tmp}
declare -r boxesArgs=$(cat ${opt_testCase} | sed -n '/^:ARGS/,+1p' | grep -v ^:INPUT | sed '1d' | tr -d '\r')
arrange_environment
arrange_test_fixtures
declare -i actualReturnCode=100
run_boxes
measure_coverage
assert_outcome
rm ${testInputFile}
rm ${testFilterFile}
rm ${testExpectationFile}
rm ${testOutputFile}
2014-09-23 23:50:36 +02:00
echo " OK"
exit 0
#EOF