mirror of
https://github.com/ascii-boxes/boxes.git
synced 2025-01-20 20:58:35 +01:00
Add test coverage measurement to black-box tests
This commit is contained in:
Thomas Jensen
parent
3c56934ddc
commit
e40fe370e2
43
Makefile
43
Makefile
@ -33,11 +33,15 @@ WIN_PCRE2_DIR = pcre2-$(WIN_PCRE2_VERSION)
|
||||
WIN_FLEX_BISON_VERSION = 2.5.24
|
||||
WIN_FLEX_BISON_DIR = flex_bison_$(WIN_FLEX_BISON_VERSION)
|
||||
|
||||
.PHONY: clean build win32 debug win32.debug win32.pcre infomsg replaceinfos test package win32.package package_common
|
||||
.PHONY: clean build cov win32 debug win32.debug win32.pcre infomsg replaceinfos test covtest \
|
||||
package win32.package package_common
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
build debug: infomsg replaceinfos
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Build
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
build cov debug: infomsg replaceinfos
|
||||
$(MAKE) -C src BOXES_PLATFORM=unix LEX=flex YACC=bison $@
|
||||
|
||||
win32: infomsg replaceinfos
|
||||
@ -64,7 +68,7 @@ win32.prereq:
|
||||
rm pcre2-$(WIN_PCRE2_VERSION).tar.gz
|
||||
rm win_flex_bison-$(WIN_FLEX_BISON_VERSION).zip
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
infomsg:
|
||||
@echo "| For compilation info please refer to the boxes compilation FAQ"
|
||||
@ -88,7 +92,10 @@ doc/boxes.1.html: doc/boxes.1
|
||||
rm -f doc/boxes.1.raw.html
|
||||
@echo "Conversion complete. Excessive manual work remains."
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Standalone Packaging (Unix / Windows)
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
$(OUT_DIR)/$(PKG_NAME).tar.gz:
|
||||
mkdir -p $(OUT_DIR)/$(PKG_NAME)/doc
|
||||
@ -116,7 +123,10 @@ package: build
|
||||
|
||||
win32.package: win32 $(OUT_DIR)/zip/$(PKG_NAME).zip
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Chocolatey Packaging (Windows)
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
choco: tools/LICENSE.txt tools/boxes.cfg tools/boxes.exe
|
||||
choco pack --version=$(BVERSION)
|
||||
@ -132,17 +142,26 @@ tools/boxes.exe: $(OUT_DIR)/boxes.exe
|
||||
|
||||
$(OUT_DIR)/boxes.exe: win32
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Test
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
test:
|
||||
cd test; ./testrunner.sh --suite
|
||||
|
||||
covtest:
|
||||
cd test; ./testrunner.sh --suite --coverage
|
||||
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
# Cleanup
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
clean:
|
||||
rm -f $(OUT_DIR)/boxes.h tools/boxes.cfg tools/LICENSE.txt tools/boxes.exe tools/README*.md boxes.portable.*.nupkg
|
||||
rm -f doc/boxes.1 doc/boxes.1.raw.html doc/boxes.1.html
|
||||
$(MAKE) -C src clean
|
||||
|
||||
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
|
||||
|
||||
test:
|
||||
cd test; ./testrunner.sh -suite
|
||||
|
||||
|
||||
#EOF
|
||||
|
||||
18
src/Makefile
18
src/Makefile
@ -37,7 +37,7 @@ ORIG_SRC = $(ORIG_GEN) $(ORIG_NORM)
|
||||
ORIG_FILES = $(ORIG_SRC) $(ORIG_HDR)
|
||||
|
||||
|
||||
.PHONY: check_dir clean build debug package flags_unix flags_win32 flags_
|
||||
.PHONY: check_dir clean build cov debug package flags_unix flags_win32 flags_
|
||||
|
||||
.NOTPARALLEL:
|
||||
|
||||
@ -51,12 +51,18 @@ $(OUT_DIR):
|
||||
mkdir $(OUT_DIR)
|
||||
|
||||
build: flags_$(BOXES_PLATFORM) | $(OUT_DIR)
|
||||
$(MAKE) -C $(OUT_DIR) -f $(SRC_DIR)/Makefile BOXES_PLATFORM=$(BOXES_PLATFORM) ALL_OBJ="$(ALL_OBJ)" \
|
||||
CFLAGS_ADDTL="-O $(CFLAGS_ADDTL)" STRIP=true flags_$(BOXES_PLATFORM) $(BOXES_EXECUTABLE_NAME)
|
||||
$(MAKE) -C $(OUT_DIR) -f $(SRC_DIR)/Makefile BOXES_PLATFORM=$(BOXES_PLATFORM) ALL_OBJ="$(ALL_OBJ)" STRIP=true \
|
||||
CFLAGS_ADDTL="-O $(CFLAGS_ADDTL)" flags_$(BOXES_PLATFORM) $(BOXES_EXECUTABLE_NAME)
|
||||
|
||||
cov: flags_$(BOXES_PLATFORM) | $(OUT_DIR)
|
||||
$(MAKE) -C $(OUT_DIR) -f $(SRC_DIR)/Makefile BOXES_PLATFORM=$(BOXES_PLATFORM) ALL_OBJ="$(ALL_OBJ)" STRIP=false \
|
||||
CFLAGS_ADDTL="-g --coverage $(CFLAGS_ADDTL)" \
|
||||
LDFLAGS_ADDTL="--coverage $(LDFLAGS_ADDTL)" \
|
||||
flags_$(BOXES_PLATFORM) $(BOXES_EXECUTABLE_NAME)
|
||||
|
||||
debug: flags_$(BOXES_PLATFORM) | $(OUT_DIR)
|
||||
$(MAKE) -C $(OUT_DIR) -f $(SRC_DIR)/Makefile BOXES_PLATFORM=$(BOXES_PLATFORM) ALL_OBJ="$(ALL_OBJ)" \
|
||||
CFLAGS_ADDTL="-g $(CFLAGS_ADDTL)" STRIP=false flags_$(BOXES_PLATFORM) $(BOXES_EXECUTABLE_NAME)
|
||||
$(MAKE) -C $(OUT_DIR) -f $(SRC_DIR)/Makefile BOXES_PLATFORM=$(BOXES_PLATFORM) ALL_OBJ="$(ALL_OBJ)" STRIP=false \
|
||||
CFLAGS_ADDTL="-g $(CFLAGS_ADDTL)" flags_$(BOXES_PLATFORM) $(BOXES_EXECUTABLE_NAME)
|
||||
|
||||
boxes: $(ALL_OBJ) | check_dir
|
||||
$(CC) $(LDFLAGS) $^ -o $@ -lunistring -lpcre2-32
|
||||
@ -107,10 +113,12 @@ shape.o: shape.c shape.h boxes.h tools.h config.h | check_dir
|
||||
tools.o: tools.c tools.h boxes.h shape.h unicode.h config.h | check_dir
|
||||
unicode.o: unicode.c unicode.h boxes.h tools.h config.h | check_dir
|
||||
|
||||
|
||||
package: $(BOXES_EXECUTABLE_NAME)
|
||||
if [ -z "$(PKG_NAME)" ] ; then exit 1 ; fi
|
||||
cp -a $(BOXES_EXECUTABLE_NAME) $(PKG_NAME)/
|
||||
|
||||
|
||||
clean:
|
||||
rm -rf $(OUT_DIR)
|
||||
rm -f core $(GEN_HDR)
|
||||
|
||||
@ -11,27 +11,67 @@
|
||||
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
# _____________________________________________________________________________________________________________________
|
||||
#
|
||||
# Low-tech test runner for boxes.
|
||||
# Test runner for the black-box tests.
|
||||
# _____________________________________________________________________________________________________________________
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo 'Usage: testrunner.sh {-suite | <testCaseFile>}'
|
||||
measureCoverage=false
|
||||
if [ $# -lt 1 -o $# -gt 2 ]; then
|
||||
echo 'Usage: testrunner.sh [--coverage] {--suite | <testCaseFile>}'
|
||||
echo ' Returns 0 for success, else non-zero'
|
||||
exit 2
|
||||
elif [ $# -eq 1 ]; then
|
||||
if [ "$1" == "--coverage" ]; then
|
||||
echo 'Usage: testrunner.sh [--coverage] {--suite | <testCaseFile>}'
|
||||
echo ' Returns 0 for success, else non-zero'
|
||||
exit 2
|
||||
fi
|
||||
elif [ $# -eq 2 ]; then
|
||||
if [ "$1" != "--coverage" -a "$2" != "--coverage" ]; then
|
||||
echo 'Usage: testrunner.sh [--coverage] {--suite | <testCaseFile>}'
|
||||
echo ' Returns 0 for success, else non-zero'
|
||||
exit 2
|
||||
else
|
||||
measureCoverage=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${PWD##*/} != "test" ]; then
|
||||
>&2 echo "Please run this script from the test folder."
|
||||
exit 2
|
||||
fi
|
||||
if [ ! -d ../out ]; then
|
||||
>&2 echo "Please run 'make' from the project root to build the executable before running the tests."
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ $measureCoverage == true ]; then
|
||||
if [ $(ls ../out/*.gcno 2>/dev/null | wc -l) -lt 1 ]; then
|
||||
>&2 echo "Binaries not instrumented. Run 'make cov' from the project root."
|
||||
exit 5
|
||||
fi
|
||||
if [ "$1" == "--suite" -o "$2" == "--suite" ]; then
|
||||
rm -f ../out/lcov-baseline.info
|
||||
fi
|
||||
if [ ! -f ../out/lcov-baseline.info ]; then
|
||||
echo "Creating coverage baseline ..."
|
||||
lcov --capture --initial --no-recursion --directory ../out --base-directory ../src \
|
||||
--exclude '*/lex.yy.c' --exclude '*/parser.c' --output-file ../out/lcov-baseline.info
|
||||
echo -e "Coverage baseline created in ../out/lcov-baseline.info\n"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Execute the entire test suite
|
||||
if [ "$1" == "-suite" ]; then
|
||||
if [ "$1" == "--suite" -o "$2" == "--suite" ]; then
|
||||
declare -i overallResult=0
|
||||
declare -i countExecuted=0
|
||||
declare -i countFailed=0
|
||||
declare tc
|
||||
for tc in *.txt; do
|
||||
$0 $tc
|
||||
if [ $measureCoverage == true ]; then
|
||||
$0 --coverage $tc
|
||||
else
|
||||
$0 $tc
|
||||
fi
|
||||
if [ $? -ne 0 ]; then
|
||||
overallResult=1
|
||||
((countFailed++))
|
||||
@ -39,11 +79,30 @@ if [ "$1" == "-suite" ]; then
|
||||
((countExecuted++))
|
||||
done
|
||||
echo "$countExecuted tests executed, $(($countExecuted-$countFailed)) successful, $countFailed failed."
|
||||
|
||||
if [ $measureCoverage == true ]; then
|
||||
echo -e "\nConsolidating test coverage ..."
|
||||
pushd ../out/test-results
|
||||
find . -name *.info | xargs printf -- '--add-tracefile %s\n' | xargs --exit \
|
||||
lcov --rc lcov_branch_coverage=1 --exclude '*/lex.yy.c' --exclude '*/parser.c' \
|
||||
--output-file ../lcov-blackbox.info --add-tracefile ../lcov-baseline.info
|
||||
popd
|
||||
echo ""
|
||||
|
||||
declare -r testReportDir=../out/coverage
|
||||
mkdir -p ${testReportDir}
|
||||
genhtml --title "Boxes / black-box tests" --branch-coverage --legend \
|
||||
--output-directory ${testReportDir} ../out/lcov-blackbox.info
|
||||
echo -e "\nTest coverage report created in ${testReportDir}/index.html"
|
||||
fi
|
||||
exit $overallResult
|
||||
fi
|
||||
|
||||
# Execute only a single test
|
||||
declare -r testCaseFile="$1"
|
||||
declare testCaseFile="$1"
|
||||
if [ "$1" == "--coverage" ]; then
|
||||
testCaseFile="$2"
|
||||
fi
|
||||
if [ ! -f $testCaseFile ]; then
|
||||
>&2 echo "Test Case '$testCaseFile' not found."
|
||||
exit 3
|
||||
@ -51,6 +110,11 @@ fi
|
||||
|
||||
echo "Running test case: $testCaseFile"
|
||||
|
||||
declare -r testCaseName=${testCaseFile:0:-4}
|
||||
if [ $measureCoverage == true ]; then
|
||||
rm -f ../out/*.gcda # remove any left-over coverage data files
|
||||
fi
|
||||
|
||||
declare sectionName
|
||||
for sectionName in :ARGS :INPUT :OUTPUT-FILTER :EXPECTED :EOF; do
|
||||
if [ $(grep -c ^$sectionName $testCaseFile) -ne 1 ]; then
|
||||
@ -99,6 +163,18 @@ else
|
||||
cat $testInputFile | eval "$boxesBinary $boxesArgs" >$testOutputFile 2>&1
|
||||
fi
|
||||
declare -ir actualReturnCode=$?
|
||||
|
||||
declare -r testResultsDir=../out/test-results/${testCaseName}
|
||||
if [ $measureCoverage == true ]; then
|
||||
mkdir -p ${testResultsDir}
|
||||
cp ../out/*.gc* ${testResultsDir}
|
||||
lcov --capture --directory ${testResultsDir} --base-directory ../src --test-name ${testCaseName} --quiet \
|
||||
--exclude '*/lex.yy.c' --exclude '*/parser.c' --rc lcov_branch_coverage=1 \
|
||||
--output-file ${testResultsDir}/coverage.info
|
||||
echo -n " Coverage: "
|
||||
lcov --summary ${testResultsDir}/coverage.info 2>&1 | grep 'lines...' | grep -oP '\d+\.\d*%'
|
||||
fi
|
||||
|
||||
cat $testOutputFile | tr -d '\r' | sed -E -f $testFilterFile | diff - $testExpectationFile
|
||||
if [ $? -ne 0 ]; then
|
||||
>&2 echo "Error in test case: $testCaseFile (top: actual; bottom: expected)"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user