From 0eb8272e9b7317e2831a0de90ed444710d26f142 Mon Sep 17 00:00:00 2001 From: Thomas Jensen Date: Sun, 24 Dec 2023 14:00:46 +0100 Subject: [PATCH] Change testrunner.sh so that coverage is only calculated for all black-box tests in total Coverage per test can be displayed using the new `--coverage-per-test` option, which implies `--coverage`. This should speed up our CI workflows. --- test/testrunner.sh | 86 ++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/test/testrunner.sh b/test/testrunner.sh index 7388c28..51bac23 100755 --- a/test/testrunner.sh +++ b/test/testrunner.sh @@ -25,13 +25,14 @@ declare -r COVERAGE_FILE=${OUT_DIR}/lcov-total.info # Command Line Options declare opt_coverage=false +declare opt_coverage_per_test=false declare opt_suite=false declare opt_testCase="" function print_usage() { - echo 'Usage: testrunner.sh [--coverage] {--suite | }' + echo 'Usage: testrunner.sh [--coverage] [--coverage-per-test] {--suite | }' echo ' Returns 0 for success, else non-zero' } @@ -49,6 +50,10 @@ function parse_arguments() opt_coverage=true shift ;; + --coverage-per-test) + opt_coverage_per_test=true + opt_coverage=true + ;; --suite) opt_suite=true shift @@ -112,6 +117,16 @@ function cov_baseline() } +function cov_args() +{ + if [ ${opt_coverage_per_test} == true ]; then + echo '--coverage-per-test' + elif [ ${opt_coverage} == true ]; then + echo '--coverage' + fi +} + + function execute_suite() { local countExecuted=0 @@ -122,18 +137,10 @@ function execute_suite() # unique and runs under ISO_8859-15. But this only happens on macOS. # So, if we run test 111 on macOS, we should run with LC_ALL=C for tc in *.txt; do - if [ ${opt_coverage} == true ]; then - if [[ $(uname) == "Darwin" ]] && [[ ${tc} == "111"* ]]; then - LC_ALL=C $0 --coverage "${tc}" - else - $0 --coverage "${tc}" - fi + if [[ $(uname) == "Darwin" ]] && [[ ${tc} == "111"* ]]; then + LC_ALL=C $0 "$(cov_args)" "${tc}" else - if [[ $(uname) == "Darwin" ]] && [[ ${tc} == "111"* ]]; then - LC_ALL=C $0 "${tc}" - else - $0 "${tc}" - fi + $0 "$(cov_args)" "${tc}" fi if [ $? -ne 0 ]; then overallResult=1 @@ -147,8 +154,8 @@ function execute_suite() function measure_coverage() { - local testResultsDir=${OUT_DIR}/test-results/${tcBaseName} if [ ${opt_coverage} == true ]; then + local testResultsDir=${OUT_DIR}/test-results/${tcBaseName} mkdir -p "${testResultsDir}" cp ${OUT_DIR}/*.gc* "${testResultsDir}" lcov --capture --directory "${testResultsDir}" --base-directory ${SRC_DIR} --test-name "${tcBaseName}" --quiet \ @@ -162,13 +169,15 @@ function measure_coverage() function consolidate_coverage() { - echo -e "\nConsolidating test coverage ..." - pushd ${OUT_DIR}/test-results || exit 1 - find . -name "*.info" | xargs printf -- '--add-tracefile %s\n' | xargs --exit \ - lcov --rc "${branchCoverage}=1" --exclude '*/lex.yy.c' --exclude '*/parser.c' \ - --output-file ../${COVERAGE_FILE} --add-tracefile ../${BASELINE_FILE} - popd || exit 1 - echo "" + if [[ ${opt_coverage} == true ]]; then + echo -e "\nConsolidating test coverage ..." + pushd ${OUT_DIR}/test-results || exit 1 + find . -name "*.info" | xargs printf -- '--add-tracefile %s\n' | xargs --exit \ + lcov --rc "${branchCoverage}=1" --exclude '*/lex.yy.c' --exclude '*/parser.c' \ + --output-file ../${COVERAGE_FILE} --add-tracefile ../${BASELINE_FILE} + popd || exit 1 + echo "" + fi } @@ -205,8 +214,8 @@ function check_mandatory_sections() function arrange_environment() { local boxesEnv="" - if [ $(grep -c "^:ENV" ${opt_testCase}) -eq 1 ]; then - boxesEnv=$(cat ${opt_testCase} | sed -n '/^:ENV/,/^:ARGS/p;' | sed '1d;$d' | tr -d '\r') + if [[ $(grep -c "^:ENV" "${opt_testCase}") -eq 1 ]]; then + boxesEnv=$(sed -n '/^:ENV/,/^:ARGS/p;' < "${opt_testCase}" | sed '1d;$d' | tr -d '\r') fi if [ -n "$boxesEnv" ]; then echo "$boxesEnv" | sed -e 's/export/\n export/g' | sed '1d' @@ -221,12 +230,12 @@ function arrange_environment() function arrange_test_fixtures() { if [ $(grep -c "^:EXPECTED-ERROR " ${opt_testCase}) -eq 1 ]; then - expectedReturnCode=$(grep "^:EXPECTED-ERROR " ${opt_testCase} | sed -e 's/:EXPECTED-ERROR //') + expectedReturnCode=$(grep "^:EXPECTED-ERROR " "${opt_testCase}" | sed -e 's/:EXPECTED-ERROR //') fi - cat ${opt_testCase} | sed -n '/^:INPUT/,/^:OUTPUT-FILTER/p;' | sed '1d;$d' | tr -d '\r' > "${testInputFile}" - cat ${opt_testCase} | sed -n '/^:OUTPUT-FILTER/,/^:EXPECTED\b.*$/p;' | sed '1d;$d' | tr -d '\r' > "${testFilterFile}" - cat ${opt_testCase} | sed -n '/^:EXPECTED/,/^:EOF/p;' | sed '1d;$d' | tr -d '\r' > "${testExpectationFile}" + cat "${opt_testCase}" | sed -n '/^:INPUT/,/^:OUTPUT-FILTER/p;' | sed '1d;$d' | tr -d '\r' > "${testInputFile}" + cat "${opt_testCase}" | sed -n '/^:OUTPUT-FILTER/,/^:EXPECTED\b.*$/p;' | sed '1d;$d' | tr -d '\r' > "${testFilterFile}" + cat "${opt_testCase}" | sed -n '/^:EXPECTED/,/^:EOF/p;' | sed '1d;$d' | tr -d '\r' > "${testExpectationFile}" } @@ -266,6 +275,7 @@ parse_arguments "$@" check_prereqs cov_baseline +declare tcBaseName=${opt_testCase%.txt} declare branchCoverage=lcov_branch_coverage if [[ $(uname) == "Darwin" ]]; then branchCoverage=branch_coverage @@ -274,8 +284,13 @@ fi # Execute the entire test suite if [ ${opt_suite} == true ]; then declare -i overallResult=0 + clear_gcda_traces execute_suite + if [ ${opt_coverage_per_test} == false ]; then + tcBaseName=black-box-all + measure_coverage + fi if [ ${opt_coverage} == true ]; then consolidate_coverage report_coverage @@ -285,8 +300,9 @@ fi # Execute only a single test echo "Running test case: ${opt_testCase}" -declare -r tcBaseName=${opt_testCase%.txt} -clear_gcda_traces +if [ ${opt_coverage_per_test} == true ]; then + clear_gcda_traces +fi check_mandatory_sections @@ -295,21 +311,23 @@ declare -r testInputFile=${opt_testCase/%.txt/.input.tmp} declare -r testExpectationFile=${opt_testCase/%.txt/.expected.tmp} declare -r testFilterFile=${opt_testCase/%.txt/.sed.tmp} declare -r testOutputFile=${opt_testCase/%.txt/.out.tmp} -declare -r boxesArgs=$(cat ${opt_testCase} | sed -n '/^:ARGS/,+1p' | grep -v ^:INPUT | sed '1d' | tr -d '\r') +declare -r boxesArgs=$(sed -n '/^:ARGS/,+1p' < "${opt_testCase}" | grep -v ^:INPUT | sed '1d' | tr -d '\r') arrange_environment arrange_test_fixtures declare -i actualReturnCode=100 run_boxes -measure_coverage +if [ ${opt_coverage_per_test} == true ]; then + measure_coverage +fi assert_outcome -rm ${testInputFile} -rm ${testFilterFile} -rm ${testExpectationFile} -rm ${testOutputFile} +rm "${testInputFile}" +rm "${testFilterFile}" +rm "${testExpectationFile}" +rm "${testOutputFile}" echo " OK" exit 0