roland/xo-umbrella2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

build: coverage working at umbrella level on osx

Browse source 
Uses llvm-cov tooling
This commit is contained in:
Roland Conybeare 2026-04-20 08:57:22 -04:00
commit 1adaf1c061
8 changed files with 276 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

6
CMakeLists.txt
View file

@ -148,6 +148,12 @@ add_subdirectory(xo-imgui)
xo_umbrella_genfacet_all(xo-genfacet-all)
# ----------------------------------------------------------------
# umbrella-wide coverage target + utest binary manifest.
# no-op unless CMAKE_BUILD_TYPE=coverage.
xo_umbrella_coverage_config()
# ----------------------------------------------------------------
# documentation. must follow add_subdirectory() for satellite projects

2
README.md
View file

@ -113,6 +113,8 @@ $ (cd .build && ctest)
Generate coverage report
```
$ .build/gen-ccov
# or:
$ cmake --build .build --target ccov
```
Html report left in ``.build/ccov/html/index.html``

3
default.nix
View file

@ -334,7 +334,6 @@ let
pkgs.cloc
pkgs.bloaty
pkgs.lcov
pkgs.catch2
pkgs.btop
@ -342,8 +341,10 @@ let
++ (if pkgs.stdenv.isLinux then [
pkgs.gdb
pkgs.strace
pkgs.lcov # gcov-format coverage post-processing
] else [
pkgs.llvmPackages_18.lldb
pkgs.llvmPackages_18.llvm # llvm-profdata, llvm-cov for source-based coverage
])
++ [
pkgs.which

19
xo-cmake/CMakeLists.txt
View file

@ -7,15 +7,31 @@ include (GNUInstallDirs)
set(XO_PROJECT_NAME xo_macros)
# LCOV_EXECUTABLE,GENHTML_EXECUTABLE: needed by xo-cmake-lcov-harness.in
# LCOV_EXECUTABLE,GENHTML_EXECUTABLE: needed by xo-cmake-lcov-harness.in (gcov toolchain)
find_program(LCOV_EXECUTABLE NAMES lcov)
find_program(GENHTML_EXECUTABLE NAMES genhtml)
# LLVM_PROFDATA_EXECUTABLE,LLVM_COV_EXECUTABLE: needed by xo-cmake-llvmcov-harness.in (llvm toolchain)
find_program(LLVM_PROFDATA_EXECUTABLE NAMES llvm-profdata)
find_program(LLVM_COV_EXECUTABLE NAMES llvm-cov)
configure_file(
${PROJECT_SOURCE_DIR}/bin/xo-cmake-lcov-harness.in
${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness
@ONLY
)
file(CHMOD ${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
configure_file(
${PROJECT_SOURCE_DIR}/bin/xo-cmake-llvmcov-harness.in
${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness
@ONLY
)
file(CHMOD ${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
configure_file(
${PROJECT_SOURCE_DIR}/bin/xo-cmake-config.in
@ -41,6 +57,7 @@ install(
install(
FILES
"${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness"
"${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness"
"${PROJECT_BINARY_DIR}/xo-cmake-config"
"${PROJECT_BINARY_DIR}/xo-build"
PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE

10
xo-cmake/bin/xo-cmake-config.in
View file

@ -1,7 +1,7 @@
#!/usr/bin/env bash
usage() {
echo "$0 [-u|--usage|-h|--help|--lcov-exe|--genhtml-exe|--lcov-harness-exe|--gen-ccov-template|--reconfigure-template|--cmake-module-path|--subsystem-list]" 1>&2
echo "$0 [-u|--usage|-h|--help|--lcov-exe|--genhtml-exe|--lcov-harness-exe|--llvmcov-harness-exe|--gen-ccov-template|--reconfigure-template|--cmake-module-path|--subsystem-list]" 1>&2
}
help() {
@ -17,7 +17,8 @@ Options:
--cmake-module-path report directory providing xo-cmake macros (will use/appear-in CMAKE_MODULE_PATH)
--lcov-exe report path to 'lcov' executable
--genhtml-exe report path to 'genhtml' executable
--lcov-harness-exe report path to 'xo-cmake-lcov-harness' executable
--lcov-harness-exe report path to 'xo-cmake-lcov-harness' executable (gcov toolchain)
--llvmcov-harness-exe report path to 'xo-cmake-llvmcov-harness' executable (llvm toolchain)
--gen-ccov-template report path to 'gen-ccov.in' template
--reconfigure-template report path to 'xo-reconfigure.in' template
--doxygen-template report path to 'Doxyfile.in' template
@ -46,6 +47,9 @@ while [[ $# > 0 ]]; do
--lcov-harness-exe)
cmd='lcov_harness_exe'
;;
--llvmcov-harness-exe)
cmd='llvmcov_harness_exe'
;;
--gen-ccov-template)
cmd='gen_ccov_template'
;;
@ -81,6 +85,8 @@ elif [[ $cmd == 'genhtml_exe' ]]; then
echo -n @GENHTML_EXECUTABLE@
elif [[ $cmd == 'lcov_harness_exe' ]]; then
echo -n @CMAKE_INSTALL_FULL_BINDIR@/xo-cmake-lcov-harness
elif [[ $cmd == 'llvmcov_harness_exe' ]]; then
echo -n @CMAKE_INSTALL_FULL_BINDIR@/xo-cmake-llvmcov-harness
elif [[ $cmd == 'gen_ccov_template' ]]; then
echo -n @CMAKE_INSTALL_FULL_DATADIR@/xo-macros/gen-ccov.in
elif [[ $cmd == 'reconfigure_template' ]]; then

118
xo-cmake/bin/xo-cmake-llvmcov-harness.in Normal file
View file

@ -0,0 +1,118 @@
#!/usr/bin/env bash
#
# xo-cmake-llvmcov-harness:
# clang/llvm source-based coverage driver.
# Parallel to xo-cmake-lcov-harness (gcov path).
#
# Pipeline:
# 1. run ctest with LLVM_PROFILE_FILE pointing into ${builddir}/ccov/raw
# 2. merge the produced .profraw files to ${outputstem}.profdata
# 3. read ${builddir}/ccov/utest-binaries.list (written by xo_umbrella_coverage_config)
# 4. emit browsable HTML under ${builddir}/ccov/html
# 5. print summary table to stdout
#
set -euo pipefail
srcdir=${1:-}
builddir=${2:-}
outputstem=${3:-}
if [[ -z "${srcdir}" || -z "${builddir}" ]]; then
echo "usage: $0 srcdir builddir [outputstem]" >&2
exit 1
fi
if [[ -z "${outputstem}" ]]; then
outputstem=${builddir}/ccov/out
fi
llvm_profdata=@LLVM_PROFDATA_EXECUTABLE@
llvm_cov=@LLVM_COV_EXECUTABLE@
for kv in "llvm-profdata:${llvm_profdata}" "llvm-cov:${llvm_cov}"; do
name=${kv%%:*}
path=${kv#*:}
if [[ -z "${path}" || "${path}" == *NOTFOUND ]]; then
echo "xo-cmake-llvmcov-harness: ${name} not found during xo-cmake build/install" >&2
exit 1
fi
done
manifest=${builddir}/ccov/utest-binaries.list
if [[ ! -f "${manifest}" ]]; then
echo "xo-cmake-llvmcov-harness: missing ${manifest}" >&2
echo "xo-cmake-llvmcov-harness: expected xo_umbrella_coverage_config() to produce it" >&2
exit 1
fi
rawdir=${builddir}/ccov/raw
htmldir=${builddir}/ccov/html
mkdir -p "${rawdir}" "${htmldir}"
# clear prior raws so we don't mix stale data from a past run
rm -f "${rawdir}"/*.profraw
# 1. run tests. LLVM_PROFILE_FILE pattern: %h=hostname %m=binary-signature %p=pid
# keeps output unique across parallel tests and across binaries.
#
# tolerate ctest failures: a crashing/asserting utest still emits .profraw
# before dying, and we want a coverage report even when some tests are broken.
# Surface the exit code at the end.
ctest_rc=0
(cd "${builddir}" && \
LLVM_PROFILE_FILE="${rawdir}/%h-%m-%p.profraw" \
ctest --output-on-failure) || ctest_rc=$?
# 2. merge .profraw -> .profdata
shopt -s nullglob
raws=("${rawdir}"/*.profraw)
if [[ ${#raws[@]} -eq 0 ]]; then
echo "xo-cmake-llvmcov-harness: no .profraw produced - was the build instrumented?" >&2
exit 1
fi
"${llvm_profdata}" merge -sparse "${raws[@]}" -o "${outputstem}.profdata"
# 3. read binary list. drop blank lines and anything not executable
# (target may be in manifest but unbuilt if 'ninja ccov' was invoked
# before a full build; defensive).
readarray -t all_bins < "${manifest}"
bins=()
for b in "${all_bins[@]}"; do
[[ -n "${b}" && -x "${b}" ]] && bins+=("${b}")
done
if [[ ${#bins[@]} -eq 0 ]]; then
echo "xo-cmake-llvmcov-harness: no executable utests found in manifest" >&2
exit 1
fi
# llvm-cov takes the first binary positionally; the rest via -object.
first=${bins[0]}
objargs=()
if [[ ${#bins[@]} -gt 1 ]]; then
for b in "${bins[@]:1}"; do objargs+=(-object "${b}"); done
fi
# exclude test code and toolchain headers from the report
ignore_re='/utest/|/nix/store/'
# 4. HTML
"${llvm_cov}" show \
-instr-profile="${outputstem}.profdata" \
-format=html \
-output-dir="${htmldir}" \
-show-line-counts-or-regions \
-ignore-filename-regex="${ignore_re}" \
"${first}" "${objargs[@]}"
# 5. summary to stdout
"${llvm_cov}" report \
-instr-profile="${outputstem}.profdata" \
-ignore-filename-regex="${ignore_re}" \
"${first}" "${objargs[@]}"
echo
echo "xo-cmake-llvmcov-harness: HTML report -> ${htmldir}/index.html"
if [[ ${ctest_rc} -ne 0 ]]; then
echo "xo-cmake-llvmcov-harness: WARNING ctest exited ${ctest_rc} - coverage is from successful tests only" >&2
fi

117
xo-cmake/cmake/xo_macros/xo_cxx.cmake
View file

@ -236,32 +236,56 @@ endmacro()
#
macro(xo_toplevel_coverage_config2)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "coverage")
#find_program(LCOV_EXECUTABLE NAMES lcov)
#find_program(GENHTML_EXECUTABLE NAMES genhtml)
# see bin/xo-cmake-lcov-harness in this repo
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --lcov-harness-exe OUTPUT_VARIABLE XO_CMAKE_LCOV_HARNESS_EXECUTABLE)
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --gen-ccov-template OUTPUT_VARIABLE XO_CMAKE_GEN_CCOV_TEMPLATE)
# select coverage toolchain by compiler family:
# GNU -> gcov (post-process with lcov/genhtml)
# Clang/AppleClang -> llvm source-based coverage (post-process with llvm-profdata/llvm-cov)
# Apple clang ships no libgcov, so the gcov path does not link on darwin.
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
set(XO_COVERAGE_TOOLCHAIN "llvm" CACHE STRING "coverage toolchain: gcov or llvm")
set(_xo_cov_compile -ggdb -Og -fprofile-instr-generate -fcoverage-mapping)
set(_xo_cov_link -fprofile-instr-generate)
else()
set(XO_COVERAGE_TOOLCHAIN "gcov" CACHE STRING "coverage toolchain: gcov or llvm")
set(_xo_cov_compile -ggdb -Og -fprofile-arcs -ftest-coverage)
set(_xo_cov_link --coverage)
endif()
message(STATUS "XO_COVERAGE_TOOLCHAIN=${XO_COVERAGE_TOOLCHAIN}")
if (NOT DEFINED PROJECT_CXX_FLAGS_COVERAGE)
# note: for clang would use -fprofile-instr-generate -fcoverage-mapping here instead and also at link time
set(PROJECT_CXX_FLAGS_COVERAGE -ggdb -Og -fprofile-arcs -ftest-coverage
set(PROJECT_CXX_FLAGS_COVERAGE ${_xo_cov_compile}
CACHE STRING "coverage c++ compiler flags")
endif()
message(STATUS "PROJECT_CXX_FLAGS_COVERAGE: coverage c++ flags are [${PROJECT_CXX_FLAGS_COVERAGE}]")
add_compile_options("$<$<CONFIG:COVERAGE>:${PROJECT_CXX_FLAGS_COVERAGE}>")
# when -DCMAKE_BUILD_TYPE=coverage, must also link executables with gcov
link_libraries("$<$<CONFIG:COVERAGE>:gcov>")
add_link_options("$<$<CONFIG:COVERAGE>:${_xo_cov_link}>")
# per-project gen-ccov: only in standalone satellite builds.
# umbrella builds delegate to xo_umbrella_coverage_config(), which reads
# the in-tree template directly (bypassing the installed xo-cmake-config).
if (NOT XO_SUBMODULE_BUILD)
# xo-cmake-config reports installed paths of the harnesses + template.
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --lcov-harness-exe OUTPUT_VARIABLE _xo_cov_lcov_harness)
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --llvmcov-harness-exe OUTPUT_VARIABLE _xo_cov_llvmcov_harness)
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --gen-ccov-template OUTPUT_VARIABLE XO_CMAKE_GEN_CCOV_TEMPLATE)
# pick the harness matching XO_COVERAGE_TOOLCHAIN
if (XO_COVERAGE_TOOLCHAIN STREQUAL "llvm")
set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${_xo_cov_llvmcov_harness})
else()
set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${_xo_cov_lcov_harness})
endif()
if("${XO_CMAKE_GEN_CCOV_TEMPLATE}" STREQUAL "")
message(ERROR "xo_toplevel_testing_config2: XO_CMAKE_GEN_CCOV_TEMPLATE not set")
message(ERROR "see xo_toplevel_testing_options2()")
message(WARNING "xo_toplevel_coverage_config2: XO_CMAKE_GEN_CCOV_TEMPLATE not set "
"(xo-cmake not installed? skipping per-project gen-ccov)")
else()
message(DEBUG "XO_CMAKE_GEN_CCOV_TEMPLATE=${XO_CMAKE_GEN_CCOV_TEMPLATE}")
configure_file(${XO_CMAKE_GEN_CCOV_TEMPLATE} ${PROJECT_BINARY_DIR}/gen-ccov @ONLY)
file(CHMOD ${PROJECT_BINARY_DIR}/gen-ccov PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
endif()
endif()
endif()
endmacro()
macro(xo_toplevel_asm_config2)
@ -301,6 +325,68 @@ function(xo_generate_reconfigure_script)
message(STATUS "Generated ${_reconfigure_script}")
endfunction()
# umbrella-level coverage wiring: emits utest-binaries.list manifest
# and the `ccov` target, driven by the singleton global utest list
# populated by xo_add_utest_executable.
#
# Call once from the umbrella CMakeLists.txt, after all add_subdirectory()
# calls for satellites. Inside a satellite-standalone build,
# xo_utest_coverage_config2 fills the equivalent role.
#
macro(xo_umbrella_coverage_config)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "coverage")
get_property(_all_utests GLOBAL PROPERTY xo_all_utest_executables)
get_property(_all_libs GLOBAL PROPERTY xo_all_shared_libraries)
list(LENGTH _all_utests _n_utests)
list(LENGTH _all_libs _n_libs)
message(STATUS "xo_umbrella_coverage_config: ${_n_utests} utests + ${_n_libs} libraries")
# manifest: one resolved binary path per line. Both utest executables
# and satellite shared libraries must appear so that llvm-cov can
# resolve coverage mappings from every instrumented object (.cpp in
# libs, templates/header code in exes). $<TARGET_FILE:...> is
# deferred until build-graph time, so file(GENERATE) is required
# (configure_file wouldn't resolve the generator expressions).
set(_bin_lines)
foreach(_t ${_all_utests} ${_all_libs})
list(APPEND _bin_lines "$<TARGET_FILE:${_t}>")
endforeach()
string(JOIN "\n" _bin_contents ${_bin_lines})
file(GENERATE
OUTPUT ${CMAKE_BINARY_DIR}/ccov/utest-binaries.list
CONTENT "${_bin_contents}\n")
# gen-ccov: use in-tree template directly, bypassing the installed
# xo-cmake-config (which doesn't exist in a from-scratch umbrella build).
# pick the harness matching XO_COVERAGE_TOOLCHAIN (set by xo_toplevel_coverage_config2).
if (XO_COVERAGE_TOOLCHAIN STREQUAL "llvm")
set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${CMAKE_BINARY_DIR}/xo-cmake/xo-cmake-llvmcov-harness)
else()
set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${CMAKE_BINARY_DIR}/xo-cmake/xo-cmake-lcov-harness)
endif()
message(STATUS "xo_umbrella_coverage_config: harness=${XO_CMAKE_COV_HARNESS_EXECUTABLE}")
set(_gen_ccov_template ${XO_UMBRELLA_SOURCE_DIR}/xo-cmake/share/xo-macros/gen-ccov.in)
if (EXISTS ${_gen_ccov_template})
configure_file(${_gen_ccov_template} ${CMAKE_BINARY_DIR}/gen-ccov @ONLY)
file(CHMOD ${CMAKE_BINARY_DIR}/gen-ccov
PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
else()
message(WARNING "xo_umbrella_coverage_config: template not found: ${_gen_ccov_template}")
endif()
# ccov target. DEPENDS on utest targets ensures they build before
# gen-ccov runs (which itself drives `ctest`).
add_custom_target(
ccov
DEPENDS ${_all_utests}
COMMAND ${CMAKE_BINARY_DIR}/gen-ccov
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
COMMENT "Generating coverage report -> ${CMAKE_BINARY_DIR}/ccov/html")
endif()
endmacro()
# target to build+install coverage report.
#
macro(xo_utest_coverage_config2)
@ -806,6 +892,11 @@ macro(xo_add_shared_library4 target projectTargets targetversion soversion sourc
APPEND
PROPERTY targets ${target})
# singleton global list of every shared library across satellites.
# consumed by xo_umbrella_coverage_config to extend the ccov binary
# manifest so llvm-cov resolves .cpp coverage in satellite .dylibs/.so.
set_property(GLOBAL APPEND PROPERTY xo_all_shared_libraries ${target})
set_target_properties(
${target}
PROPERTIES
@ -983,6 +1074,10 @@ macro(xo_add_utest_executable target sources)
PROPERTY targets ${target})
add_dependencies(all_utest_executables_${PROJECT_NAME} ${target})
# singleton global list of every utest target: survives across project() scopes,
# so umbrella and standalone-satellite builds can query one source of truth
set_property(GLOBAL APPEND PROPERTY xo_all_utest_executables ${target})
endmacro()
# ----------------------------------------------------------------

31
xo-cmake/share/xo-macros/gen-ccov.in
View file

@ -3,28 +3,17 @@
srcdir=@PROJECT_SOURCE_DIR@
builddir=@PROJECT_BINARY_DIR@
#lcov=@LCOV_EXECUTABLE@
#genhtml=@GENHTML_EXECUTABLE@
#
#if [[ $lcov == "LCOV_EXECUTABLE-NOTFOUND" ]]; then
# echo "gen-ccov: lcov executable not found"
# exit 1
#fi
#
#if [[ $genhtml == "GENHTML_EXECUTABLE-NOTFOUND" ]]; then
# echo "gen-ccov: genhtml executable not found"
# exit 1
#fi
# path to the toolchain-specific harness. Configured by either
# xo_toplevel_coverage_config2 (standalone satellite build) or
# xo_umbrella_coverage_config (umbrella build). Resolves to one of:
# xo-cmake-lcov-harness (gcov toolchain)
# xo-cmake-llvmcov-harness (llvm toolchain)
covharness=@XO_CMAKE_COV_HARNESS_EXECUTABLE@
lcovharness=@XO_CMAKE_LCOV_HARNESS_EXECUTABLE@
if [[ -z $lcovharness ]]; then
echo "gen-ccov: lcov-harness executable (XO_CMAKE_LCOV_HARNESS_EXECUTABLE) not configured"
echo "gen-ccov: expect value of path/to/xo-cmake-config --lcov-harness-exe"
echo "gen-ccov: stored in XO_CMAKE_LCOV_HARNESS_EXECUTABLE by xo_toplevel_testing_options2()"
if [[ -z $covharness ]]; then
echo "gen-ccov: coverage harness (XO_CMAKE_COV_HARNESS_EXECUTABLE) not configured"
echo "gen-ccov: should be set by xo_umbrella_coverage_config or xo_toplevel_coverage_config2"
exit 1
fi
# TODO: allow providing LCOV_EXECUTABLE GENHTML_EXECUTABLE here
$lcovharness $srcdir $builddir
$covharness $srcdir $builddir