diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst index 64c0ec7428a2..26a536ef4cd6 100644 --- a/Documentation/dev-tools/kselftest.rst +++ b/Documentation/dev-tools/kselftest.rst @@ -133,6 +133,44 @@ build successfully; make will abort as soon as any target fails:: This applies to both the ``all`` and ``install`` targets and is useful in CI environments where a silent partial build would be misleading. +Recording suite build results +============================= + +Since the build succeeds as long as one target builds, the exit status of +make does not show which targets failed. Set ``KSFT_BUILD_REPORT`` to the +absolute path of a file to have the ``all`` and ``install`` targets append +a tab-separated record for each target they process:: + + $ : > /tmp/ksft-build.tsv + $ make -C tools/testing/selftests O=/tmp/kernel-build \ + KSFT_BUILD_REPORT=/tmp/ksft-build.tsv install + +Each record holds the phase (``all`` or ``install``), a name and the exit +status of that step. Recording does not change which targets are built or +installed, or the exit status of make, also with ``FORCE_TARGETS=1``. The +caller creates or truncates the report before a build; the rules only +append to it. Nothing is recorded by ``make -n``, ``-t`` or ``-q``. + +The same name can appear several times: ``install`` builds the targets +again, and generating ``kselftest-list.txt`` is recorded as part of the +``install`` phase. Targets that are added automatically as installation +dependencies of other targets, such as ``net/lib``, are recorded too. +Suites that are built from several subdirectories may also record each +subdirectory under its path, for example ``arm64/tags``; a failure there +means that the suite was built or installed only partially. A record +belongs to the target with the longest name that is equal to the record +name or followed by ``/`` in it. + +A target was built and installed successfully only if all of its records +have a zero status. Missing records do not show that a target was built. +A successful target that installs no tests has no entries in +``kselftest-list.txt``. Install the kernel headers with ``make headers`` +before building: a suite that fails because the headers are missing is +recorded as failed even if a later step installs them. + +``KSFT_BUILD_REPORT_VERSION`` in tools/testing/selftests/Makefile gives the +version of this format, currently ``1``. + Running the full range hotplug selftests ======================================== diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 62a24e742be6..d4c71ad20bbb 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -171,6 +171,10 @@ endif # of the targets gets built. FORCE_TARGETS ?= +# Remember dry runs for the build report before MAKEFLAGS may be cleared. +KSFT_BUILD_REPORT_DRY_RUN := $(strip $(foreach f,n t q,$(findstring $(f),$(firstword -$(MAKEFLAGS))))) +export KSFT_BUILD_REPORT_DRY_RUN + # Clear LDFLAGS and MAKEFLAGS when implicit rules are missing. This provides # implicit rules to sub-test Makefiles which avoids build failures in test # Makefile that don't have explicit build rules. @@ -215,6 +219,10 @@ ARCH ?= $(SUBARCH) export BUILD export KHDR_INCLUDES +# Version of the optional KSFT_BUILD_REPORT format written by these rules. +KSFT_BUILD_REPORT_VERSION := 1 +include $(top_srcdir)/tools/testing/selftests/build_report.mk + # set default goal to all, so make without a target runs all, even when # all isn't the first target in the file. .DEFAULT_GOAL := all @@ -223,11 +231,13 @@ all: @ret=1; \ for TARGET in $(TARGETS) $(INSTALL_DEP_TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ - mkdir $$BUILD_TARGET -p; \ + mkdir -p $$BUILD_TARGET && \ $(MAKE) OUTPUT=$$BUILD_TARGET -C $$TARGET \ - O=$(abs_objtree) \ - $(if $(FORCE_TARGETS),|| exit); \ - [ $$? -eq 0 ] && ret=0; \ + O=$(abs_objtree); \ + status=$$?; \ + $(call ksft_record_build_status,all,$$TARGET); \ + $(if $(FORCE_TARGETS),[ $$status -eq 0 ] || exit $$status;) \ + [ $$status -eq 0 ] && ret=0; \ done; exit $$ret; run_tests: all @@ -284,7 +294,7 @@ ifdef INSTALL_PATH install -m 744 kselftest/ktap_helpers.sh $(INSTALL_PATH)/kselftest/ install -m 744 kselftest/ksft.py $(INSTALL_PATH)/kselftest/ install -m 744 run_kselftest.sh $(INSTALL_PATH)/ - rm -f $(TEST_LIST) + : > $(TEST_LIST) @ret=1; \ for TARGET in $(TARGETS) $(INSTALL_DEP_TARGETS); do \ BUILD_TARGET=$$BUILD/$$TARGET; \ @@ -292,9 +302,11 @@ ifdef INSTALL_PATH INSTALL_PATH=$(INSTALL_PATH)/$$TARGET \ SRC_PATH=$(shell readlink -e $$(pwd)) \ OBJ_PATH=$(INSTALL_PATH) \ - O=$(abs_objtree) \ - $(if $(FORCE_TARGETS),|| exit); \ - [ $$? -eq 0 ] && ret=0; \ + O=$(abs_objtree); \ + status=$$?; \ + $(call ksft_record_build_status,install,$$TARGET); \ + $(if $(FORCE_TARGETS),[ $$status -eq 0 ] || exit $$status;) \ + [ $$status -eq 0 ] && ret=0; \ done; exit $$ret; @@ -308,6 +320,9 @@ ifdef INSTALL_PATH printf "Emit Tests for $$TARGET\n"; \ $(MAKE) -s --no-print-directory OUTPUT=$$BUILD_TARGET COLLECTION=$$TARGET \ -C $$TARGET emit_tests >> $(TEST_LIST); \ + status=$$?; \ + $(call ksft_record_build_status,install,$$TARGET); \ + (exit $$status); \ done; @VERSION=$$(git describe HEAD 2>/dev/null); \ if [ -n "$$VERSION" ]; then \ diff --git a/tools/testing/selftests/build_report.mk b/tools/testing/selftests/build_report.mk new file mode 100644 index 000000000000..047c239925e1 --- /dev/null +++ b/tools/testing/selftests/build_report.mk @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Optional per-suite build report, see "Recording suite build results" in +# Documentation/dev-tools/kselftest.rst. Callers set KSFT_BUILD_REPORT to an +# absolute filename and use ksft_record_build_status from a shell loop that +# has stored the exit status of the recorded step in $status. + +ifneq (x$(KSFT_BUILD_REPORT)x,xx) +ifeq ($(filter x/%,$(firstword x$(KSFT_BUILD_REPORT))),) +$(error KSFT_BUILD_REPORT must be an absolute path: '$(KSFT_BUILD_REPORT)') +endif +endif +export KSFT_BUILD_REPORT + +ksft_build_report_dry_run := $(KSFT_BUILD_REPORT_DRY_RUN) \ + $(foreach f,n t q,$(findstring $(f),$(firstword -$(MAKEFLAGS)))) + +ifneq ($(strip $(ksft_build_report_dry_run)),) +ksft_record_build_status = : +else +define ksft_record_build_status +if [ -n "$$KSFT_BUILD_REPORT" ]; then \ + printf '%s\t%s\t%d\n' '$(1)' "$(2)" "$$status" >> "$$KSFT_BUILD_REPORT" || exit 1; \ +fi +endef +endif