diff --git a/CMakeLists.txt b/CMakeLists.txt index 92cd55e0..17b6cafb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,12 @@ add_subdirectory(xo-imgui) xo_umbrella_genfacet_all(xo-genfacet-all) +# ---------------------------------------------------------------- +# umbrella-wide coverage target + utest binary manifest. +# no-op unless CMAKE_BUILD_TYPE=coverage. + +xo_umbrella_coverage_config() + # ---------------------------------------------------------------- # documentation. must follow add_subdirectory() for satellite projects diff --git a/README.md b/README.md index 1a3441c6..9b921011 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,8 @@ $ (cd .build && ctest) Generate coverage report ``` $ .build/gen-ccov +# or: +$ cmake --build .build --target ccov ``` Html report left in ``.build/ccov/html/index.html`` diff --git a/default.nix b/default.nix index 4fb4f388..19d55283 100644 --- a/default.nix +++ b/default.nix @@ -334,7 +334,6 @@ let pkgs.cloc pkgs.bloaty - pkgs.lcov pkgs.catch2 pkgs.btop @@ -342,8 +341,10 @@ let ++ (if pkgs.stdenv.isLinux then [ pkgs.gdb pkgs.strace + pkgs.lcov # gcov-format coverage post-processing ] else [ pkgs.llvmPackages_18.lldb + pkgs.llvmPackages_18.llvm # llvm-profdata, llvm-cov for source-based coverage ]) ++ [ pkgs.which diff --git a/xo-cmake/CMakeLists.txt b/xo-cmake/CMakeLists.txt index 51f2a45a..b54c712c 100644 --- a/xo-cmake/CMakeLists.txt +++ b/xo-cmake/CMakeLists.txt @@ -7,15 +7,31 @@ include (GNUInstallDirs) set(XO_PROJECT_NAME xo_macros) -# LCOV_EXECUTABLE,GENHTML_EXECUTABLE: needed by xo-cmake-lcov-harness.in +# LCOV_EXECUTABLE,GENHTML_EXECUTABLE: needed by xo-cmake-lcov-harness.in (gcov toolchain) find_program(LCOV_EXECUTABLE NAMES lcov) find_program(GENHTML_EXECUTABLE NAMES genhtml) +# LLVM_PROFDATA_EXECUTABLE,LLVM_COV_EXECUTABLE: needed by xo-cmake-llvmcov-harness.in (llvm toolchain) +find_program(LLVM_PROFDATA_EXECUTABLE NAMES llvm-profdata) +find_program(LLVM_COV_EXECUTABLE NAMES llvm-cov) + configure_file( ${PROJECT_SOURCE_DIR}/bin/xo-cmake-lcov-harness.in ${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness @ONLY ) +file(CHMOD ${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + +configure_file( + ${PROJECT_SOURCE_DIR}/bin/xo-cmake-llvmcov-harness.in + ${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness + @ONLY + ) +file(CHMOD ${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) configure_file( ${PROJECT_SOURCE_DIR}/bin/xo-cmake-config.in @@ -41,6 +57,7 @@ install( install( FILES "${PROJECT_BINARY_DIR}/xo-cmake-lcov-harness" + "${PROJECT_BINARY_DIR}/xo-cmake-llvmcov-harness" "${PROJECT_BINARY_DIR}/xo-cmake-config" "${PROJECT_BINARY_DIR}/xo-build" PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE diff --git a/xo-cmake/bin/xo-cmake-config.in b/xo-cmake/bin/xo-cmake-config.in index 8cb3f39d..5a501a98 100755 --- a/xo-cmake/bin/xo-cmake-config.in +++ b/xo-cmake/bin/xo-cmake-config.in @@ -1,7 +1,7 @@ #!/usr/bin/env bash usage() { - echo "$0 [-u|--usage|-h|--help|--lcov-exe|--genhtml-exe|--lcov-harness-exe|--gen-ccov-template|--reconfigure-template|--cmake-module-path|--subsystem-list]" 1>&2 + echo "$0 [-u|--usage|-h|--help|--lcov-exe|--genhtml-exe|--lcov-harness-exe|--llvmcov-harness-exe|--gen-ccov-template|--reconfigure-template|--cmake-module-path|--subsystem-list]" 1>&2 } help() { @@ -17,7 +17,8 @@ Options: --cmake-module-path report directory providing xo-cmake macros (will use/appear-in CMAKE_MODULE_PATH) --lcov-exe report path to 'lcov' executable --genhtml-exe report path to 'genhtml' executable - --lcov-harness-exe report path to 'xo-cmake-lcov-harness' executable + --lcov-harness-exe report path to 'xo-cmake-lcov-harness' executable (gcov toolchain) + --llvmcov-harness-exe report path to 'xo-cmake-llvmcov-harness' executable (llvm toolchain) --gen-ccov-template report path to 'gen-ccov.in' template --reconfigure-template report path to 'xo-reconfigure.in' template --doxygen-template report path to 'Doxyfile.in' template @@ -46,6 +47,9 @@ while [[ $# > 0 ]]; do --lcov-harness-exe) cmd='lcov_harness_exe' ;; + --llvmcov-harness-exe) + cmd='llvmcov_harness_exe' + ;; --gen-ccov-template) cmd='gen_ccov_template' ;; @@ -81,6 +85,8 @@ elif [[ $cmd == 'genhtml_exe' ]]; then echo -n @GENHTML_EXECUTABLE@ elif [[ $cmd == 'lcov_harness_exe' ]]; then echo -n @CMAKE_INSTALL_FULL_BINDIR@/xo-cmake-lcov-harness +elif [[ $cmd == 'llvmcov_harness_exe' ]]; then + echo -n @CMAKE_INSTALL_FULL_BINDIR@/xo-cmake-llvmcov-harness elif [[ $cmd == 'gen_ccov_template' ]]; then echo -n @CMAKE_INSTALL_FULL_DATADIR@/xo-macros/gen-ccov.in elif [[ $cmd == 'reconfigure_template' ]]; then diff --git a/xo-cmake/bin/xo-cmake-llvmcov-harness.in b/xo-cmake/bin/xo-cmake-llvmcov-harness.in new file mode 100644 index 00000000..13a47601 --- /dev/null +++ b/xo-cmake/bin/xo-cmake-llvmcov-harness.in @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# +# xo-cmake-llvmcov-harness: +# clang/llvm source-based coverage driver. +# Parallel to xo-cmake-lcov-harness (gcov path). +# +# Pipeline: +# 1. run ctest with LLVM_PROFILE_FILE pointing into ${builddir}/ccov/raw +# 2. merge the produced .profraw files to ${outputstem}.profdata +# 3. read ${builddir}/ccov/utest-binaries.list (written by xo_umbrella_coverage_config) +# 4. emit browsable HTML under ${builddir}/ccov/html +# 5. print summary table to stdout +# + +set -euo pipefail + +srcdir=${1:-} +builddir=${2:-} +outputstem=${3:-} + +if [[ -z "${srcdir}" || -z "${builddir}" ]]; then + echo "usage: $0 srcdir builddir [outputstem]" >&2 + exit 1 +fi + +if [[ -z "${outputstem}" ]]; then + outputstem=${builddir}/ccov/out +fi + +llvm_profdata=@LLVM_PROFDATA_EXECUTABLE@ +llvm_cov=@LLVM_COV_EXECUTABLE@ + +for kv in "llvm-profdata:${llvm_profdata}" "llvm-cov:${llvm_cov}"; do + name=${kv%%:*} + path=${kv#*:} + if [[ -z "${path}" || "${path}" == *NOTFOUND ]]; then + echo "xo-cmake-llvmcov-harness: ${name} not found during xo-cmake build/install" >&2 + exit 1 + fi +done + +manifest=${builddir}/ccov/utest-binaries.list +if [[ ! -f "${manifest}" ]]; then + echo "xo-cmake-llvmcov-harness: missing ${manifest}" >&2 + echo "xo-cmake-llvmcov-harness: expected xo_umbrella_coverage_config() to produce it" >&2 + exit 1 +fi + +rawdir=${builddir}/ccov/raw +htmldir=${builddir}/ccov/html +mkdir -p "${rawdir}" "${htmldir}" + +# clear prior raws so we don't mix stale data from a past run +rm -f "${rawdir}"/*.profraw + +# 1. run tests. LLVM_PROFILE_FILE pattern: %h=hostname %m=binary-signature %p=pid +# keeps output unique across parallel tests and across binaries. +# +# tolerate ctest failures: a crashing/asserting utest still emits .profraw +# before dying, and we want a coverage report even when some tests are broken. +# Surface the exit code at the end. +ctest_rc=0 +(cd "${builddir}" && \ + LLVM_PROFILE_FILE="${rawdir}/%h-%m-%p.profraw" \ + ctest --output-on-failure) || ctest_rc=$? + +# 2. merge .profraw -> .profdata +shopt -s nullglob +raws=("${rawdir}"/*.profraw) +if [[ ${#raws[@]} -eq 0 ]]; then + echo "xo-cmake-llvmcov-harness: no .profraw produced - was the build instrumented?" >&2 + exit 1 +fi +"${llvm_profdata}" merge -sparse "${raws[@]}" -o "${outputstem}.profdata" + +# 3. read binary list. drop blank lines and anything not executable +# (target may be in manifest but unbuilt if 'ninja ccov' was invoked +# before a full build; defensive). +readarray -t all_bins < "${manifest}" +bins=() +for b in "${all_bins[@]}"; do + [[ -n "${b}" && -x "${b}" ]] && bins+=("${b}") +done +if [[ ${#bins[@]} -eq 0 ]]; then + echo "xo-cmake-llvmcov-harness: no executable utests found in manifest" >&2 + exit 1 +fi + +# llvm-cov takes the first binary positionally; the rest via -object. +first=${bins[0]} +objargs=() +if [[ ${#bins[@]} -gt 1 ]]; then + for b in "${bins[@]:1}"; do objargs+=(-object "${b}"); done +fi + +# exclude test code and toolchain headers from the report +ignore_re='/utest/|/nix/store/' + +# 4. HTML +"${llvm_cov}" show \ + -instr-profile="${outputstem}.profdata" \ + -format=html \ + -output-dir="${htmldir}" \ + -show-line-counts-or-regions \ + -ignore-filename-regex="${ignore_re}" \ + "${first}" "${objargs[@]}" + +# 5. summary to stdout +"${llvm_cov}" report \ + -instr-profile="${outputstem}.profdata" \ + -ignore-filename-regex="${ignore_re}" \ + "${first}" "${objargs[@]}" + +echo +echo "xo-cmake-llvmcov-harness: HTML report -> ${htmldir}/index.html" +if [[ ${ctest_rc} -ne 0 ]]; then + echo "xo-cmake-llvmcov-harness: WARNING ctest exited ${ctest_rc} - coverage is from successful tests only" >&2 +fi diff --git a/xo-cmake/cmake/xo_macros/xo_cxx.cmake b/xo-cmake/cmake/xo_macros/xo_cxx.cmake index 52169a3d..6d10beb7 100644 --- a/xo-cmake/cmake/xo_macros/xo_cxx.cmake +++ b/xo-cmake/cmake/xo_macros/xo_cxx.cmake @@ -236,30 +236,54 @@ endmacro() # macro(xo_toplevel_coverage_config2) if ("${CMAKE_BUILD_TYPE}" STREQUAL "coverage") - #find_program(LCOV_EXECUTABLE NAMES lcov) - #find_program(GENHTML_EXECUTABLE NAMES genhtml) - # see bin/xo-cmake-lcov-harness in this repo - execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --lcov-harness-exe OUTPUT_VARIABLE XO_CMAKE_LCOV_HARNESS_EXECUTABLE) - execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --gen-ccov-template OUTPUT_VARIABLE XO_CMAKE_GEN_CCOV_TEMPLATE) + # select coverage toolchain by compiler family: + # GNU -> gcov (post-process with lcov/genhtml) + # Clang/AppleClang -> llvm source-based coverage (post-process with llvm-profdata/llvm-cov) + # Apple clang ships no libgcov, so the gcov path does not link on darwin. + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set(XO_COVERAGE_TOOLCHAIN "llvm" CACHE STRING "coverage toolchain: gcov or llvm") + set(_xo_cov_compile -ggdb -Og -fprofile-instr-generate -fcoverage-mapping) + set(_xo_cov_link -fprofile-instr-generate) + else() + set(XO_COVERAGE_TOOLCHAIN "gcov" CACHE STRING "coverage toolchain: gcov or llvm") + set(_xo_cov_compile -ggdb -Og -fprofile-arcs -ftest-coverage) + set(_xo_cov_link --coverage) + endif() + message(STATUS "XO_COVERAGE_TOOLCHAIN=${XO_COVERAGE_TOOLCHAIN}") if (NOT DEFINED PROJECT_CXX_FLAGS_COVERAGE) - # note: for clang would use -fprofile-instr-generate -fcoverage-mapping here instead and also at link time - set(PROJECT_CXX_FLAGS_COVERAGE -ggdb -Og -fprofile-arcs -ftest-coverage + set(PROJECT_CXX_FLAGS_COVERAGE ${_xo_cov_compile} CACHE STRING "coverage c++ compiler flags") endif() message(STATUS "PROJECT_CXX_FLAGS_COVERAGE: coverage c++ flags are [${PROJECT_CXX_FLAGS_COVERAGE}]") add_compile_options("$<$:${PROJECT_CXX_FLAGS_COVERAGE}>") - # when -DCMAKE_BUILD_TYPE=coverage, must also link executables with gcov - link_libraries("$<$:gcov>") + add_link_options("$<$:${_xo_cov_link}>") - if("${XO_CMAKE_GEN_CCOV_TEMPLATE}" STREQUAL "") - message(ERROR "xo_toplevel_testing_config2: XO_CMAKE_GEN_CCOV_TEMPLATE not set") - message(ERROR "see xo_toplevel_testing_options2()") - else() - message(DEBUG "XO_CMAKE_GEN_CCOV_TEMPLATE=${XO_CMAKE_GEN_CCOV_TEMPLATE}") - configure_file(${XO_CMAKE_GEN_CCOV_TEMPLATE} ${PROJECT_BINARY_DIR}/gen-ccov @ONLY) - file(CHMOD ${PROJECT_BINARY_DIR}/gen-ccov PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + # per-project gen-ccov: only in standalone satellite builds. + # umbrella builds delegate to xo_umbrella_coverage_config(), which reads + # the in-tree template directly (bypassing the installed xo-cmake-config). + if (NOT XO_SUBMODULE_BUILD) + # xo-cmake-config reports installed paths of the harnesses + template. + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --lcov-harness-exe OUTPUT_VARIABLE _xo_cov_lcov_harness) + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --llvmcov-harness-exe OUTPUT_VARIABLE _xo_cov_llvmcov_harness) + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --gen-ccov-template OUTPUT_VARIABLE XO_CMAKE_GEN_CCOV_TEMPLATE) + + # pick the harness matching XO_COVERAGE_TOOLCHAIN + if (XO_COVERAGE_TOOLCHAIN STREQUAL "llvm") + set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${_xo_cov_llvmcov_harness}) + else() + set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${_xo_cov_lcov_harness}) + endif() + + if("${XO_CMAKE_GEN_CCOV_TEMPLATE}" STREQUAL "") + message(WARNING "xo_toplevel_coverage_config2: XO_CMAKE_GEN_CCOV_TEMPLATE not set " + "(xo-cmake not installed? skipping per-project gen-ccov)") + else() + message(DEBUG "XO_CMAKE_GEN_CCOV_TEMPLATE=${XO_CMAKE_GEN_CCOV_TEMPLATE}") + configure_file(${XO_CMAKE_GEN_CCOV_TEMPLATE} ${PROJECT_BINARY_DIR}/gen-ccov @ONLY) + file(CHMOD ${PROJECT_BINARY_DIR}/gen-ccov PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + endif() endif() endif() endmacro() @@ -301,6 +325,68 @@ function(xo_generate_reconfigure_script) message(STATUS "Generated ${_reconfigure_script}") endfunction() +# umbrella-level coverage wiring: emits utest-binaries.list manifest +# and the `ccov` target, driven by the singleton global utest list +# populated by xo_add_utest_executable. +# +# Call once from the umbrella CMakeLists.txt, after all add_subdirectory() +# calls for satellites. Inside a satellite-standalone build, +# xo_utest_coverage_config2 fills the equivalent role. +# +macro(xo_umbrella_coverage_config) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "coverage") + get_property(_all_utests GLOBAL PROPERTY xo_all_utest_executables) + get_property(_all_libs GLOBAL PROPERTY xo_all_shared_libraries) + list(LENGTH _all_utests _n_utests) + list(LENGTH _all_libs _n_libs) + message(STATUS "xo_umbrella_coverage_config: ${_n_utests} utests + ${_n_libs} libraries") + + # manifest: one resolved binary path per line. Both utest executables + # and satellite shared libraries must appear so that llvm-cov can + # resolve coverage mappings from every instrumented object (.cpp in + # libs, templates/header code in exes). $ is + # deferred until build-graph time, so file(GENERATE) is required + # (configure_file wouldn't resolve the generator expressions). + set(_bin_lines) + foreach(_t ${_all_utests} ${_all_libs}) + list(APPEND _bin_lines "$") + endforeach() + string(JOIN "\n" _bin_contents ${_bin_lines}) + file(GENERATE + OUTPUT ${CMAKE_BINARY_DIR}/ccov/utest-binaries.list + CONTENT "${_bin_contents}\n") + + # gen-ccov: use in-tree template directly, bypassing the installed + # xo-cmake-config (which doesn't exist in a from-scratch umbrella build). + # pick the harness matching XO_COVERAGE_TOOLCHAIN (set by xo_toplevel_coverage_config2). + if (XO_COVERAGE_TOOLCHAIN STREQUAL "llvm") + set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${CMAKE_BINARY_DIR}/xo-cmake/xo-cmake-llvmcov-harness) + else() + set(XO_CMAKE_COV_HARNESS_EXECUTABLE ${CMAKE_BINARY_DIR}/xo-cmake/xo-cmake-lcov-harness) + endif() + message(STATUS "xo_umbrella_coverage_config: harness=${XO_CMAKE_COV_HARNESS_EXECUTABLE}") + + set(_gen_ccov_template ${XO_UMBRELLA_SOURCE_DIR}/xo-cmake/share/xo-macros/gen-ccov.in) + if (EXISTS ${_gen_ccov_template}) + configure_file(${_gen_ccov_template} ${CMAKE_BINARY_DIR}/gen-ccov @ONLY) + file(CHMOD ${CMAKE_BINARY_DIR}/gen-ccov + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + else() + message(WARNING "xo_umbrella_coverage_config: template not found: ${_gen_ccov_template}") + endif() + + # ccov target. DEPENDS on utest targets ensures they build before + # gen-ccov runs (which itself drives `ctest`). + add_custom_target( + ccov + DEPENDS ${_all_utests} + COMMAND ${CMAKE_BINARY_DIR}/gen-ccov + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Generating coverage report -> ${CMAKE_BINARY_DIR}/ccov/html") + endif() +endmacro() + # target to build+install coverage report. # macro(xo_utest_coverage_config2) @@ -806,6 +892,11 @@ macro(xo_add_shared_library4 target projectTargets targetversion soversion sourc APPEND PROPERTY targets ${target}) + # singleton global list of every shared library across satellites. + # consumed by xo_umbrella_coverage_config to extend the ccov binary + # manifest so llvm-cov resolves .cpp coverage in satellite .dylibs/.so. + set_property(GLOBAL APPEND PROPERTY xo_all_shared_libraries ${target}) + set_target_properties( ${target} PROPERTIES @@ -983,6 +1074,10 @@ macro(xo_add_utest_executable target sources) PROPERTY targets ${target}) add_dependencies(all_utest_executables_${PROJECT_NAME} ${target}) + # singleton global list of every utest target: survives across project() scopes, + # so umbrella and standalone-satellite builds can query one source of truth + set_property(GLOBAL APPEND PROPERTY xo_all_utest_executables ${target}) + endmacro() # ---------------------------------------------------------------- diff --git a/xo-cmake/share/xo-macros/gen-ccov.in b/xo-cmake/share/xo-macros/gen-ccov.in index c81c3f53..1d51e688 100644 --- a/xo-cmake/share/xo-macros/gen-ccov.in +++ b/xo-cmake/share/xo-macros/gen-ccov.in @@ -3,28 +3,17 @@ srcdir=@PROJECT_SOURCE_DIR@ builddir=@PROJECT_BINARY_DIR@ -#lcov=@LCOV_EXECUTABLE@ -#genhtml=@GENHTML_EXECUTABLE@ -# -#if [[ $lcov == "LCOV_EXECUTABLE-NOTFOUND" ]]; then -# echo "gen-ccov: lcov executable not found" -# exit 1 -#fi -# -#if [[ $genhtml == "GENHTML_EXECUTABLE-NOTFOUND" ]]; then -# echo "gen-ccov: genhtml executable not found" -# exit 1 -#fi +# path to the toolchain-specific harness. Configured by either +# xo_toplevel_coverage_config2 (standalone satellite build) or +# xo_umbrella_coverage_config (umbrella build). Resolves to one of: +# xo-cmake-lcov-harness (gcov toolchain) +# xo-cmake-llvmcov-harness (llvm toolchain) +covharness=@XO_CMAKE_COV_HARNESS_EXECUTABLE@ -lcovharness=@XO_CMAKE_LCOV_HARNESS_EXECUTABLE@ - -if [[ -z $lcovharness ]]; then - echo "gen-ccov: lcov-harness executable (XO_CMAKE_LCOV_HARNESS_EXECUTABLE) not configured" - echo "gen-ccov: expect value of path/to/xo-cmake-config --lcov-harness-exe" - echo "gen-ccov: stored in XO_CMAKE_LCOV_HARNESS_EXECUTABLE by xo_toplevel_testing_options2()" +if [[ -z $covharness ]]; then + echo "gen-ccov: coverage harness (XO_CMAKE_COV_HARNESS_EXECUTABLE) not configured" + echo "gen-ccov: should be set by xo_umbrella_coverage_config or xo_toplevel_coverage_config2" exit 1 fi -# TODO: allow providing LCOV_EXECUTABLE GENHTML_EXECUTABLE here - -$lcovharness $srcdir $builddir +$covharness $srcdir $builddir