Merge pull request #1919 from giuseppe/add-coverage

test: add new target "coverage"

Author: giuseppe

Makefile.am

@@ -86,13 +86,22 @@ libocispec/libocispec.la:
 
 libcrun_la_SOURCES = $(libcrun_SOURCES)
 libcrun_la_CFLAGS = -I $(abs_top_builddir)/libocispec/src -I $(abs_top_srcdir)/libocispec/src -fvisibility=hidden
-libcrun_la_LIBADD = libocispec/libocispec.la $(FOUND_LIBS) $(maybe_libyajl.la)
+if ENABLE_COVERAGE
+libcrun_la_CFLAGS += $(COVERAGE_CFLAGS)
+libcrun_la_LDFLAGS = -Wl,--version-script=$(abs_top_srcdir)/libcrun.lds $(COVERAGE_LDFLAGS)
+else
 libcrun_la_LDFLAGS = -Wl,--version-script=$(abs_top_srcdir)/libcrun.lds
+endif
+libcrun_la_LIBADD = libocispec/libocispec.la $(FOUND_LIBS) $(maybe_libyajl.la)
 
 # build a version with all the symbols visible for testing
 if BUILD_TESTS
 libcrun_testing_la_SOURCES = $(libcrun_SOURCES)
 libcrun_testing_la_CFLAGS = -I $(abs_top_builddir)/libocispec/src -I $(abs_top_srcdir)/libocispec/src -fvisibility=default
+if ENABLE_COVERAGE
+libcrun_testing_la_CFLAGS += $(COVERAGE_CFLAGS)
+libcrun_testing_la_LDFLAGS = $(COVERAGE_LDFLAGS)
+endif
 libcrun_testing_la_LIBADD = libocispec/libocispec.la $(maybe_libyajl.la)
 endif
 
@@ -137,16 +146,27 @@ dist-luarock: $(LUACRUN_ROCK)
 endif
 
 crun_CFLAGS = -I $(abs_top_builddir)/libocispec/src -I $(abs_top_srcdir)/libocispec/src -D CRUN_LIBDIR="\"$(CRUN_LIBDIR)\""
+if ENABLE_COVERAGE
+crun_CFLAGS += $(COVERAGE_CFLAGS)
+endif
 crun_SOURCES = src/crun.c src/run.c src/delete.c src/kill.c src/pause.c src/unpause.c src/oci_features.c src/spec.c \
 		src/exec.c src/list.c src/create.c src/start.c src/state.c src/update.c src/ps.c \
 		src/checkpoint.c src/restore.c src/mounts.c src/run_create.c
 
 if DYNLOAD_LIBCRUN
+if ENABLE_COVERAGE
+crun_LDFLAGS = -Wl,--unresolved-symbols=ignore-all $(CRUN_LDFLAGS) $(COVERAGE_LDFLAGS)
+else
 crun_LDFLAGS = -Wl,--unresolved-symbols=ignore-all $(CRUN_LDFLAGS)
+endif
 else
 crun_LDADD = libcrun.la $(FOUND_LIBS) $(maybe_libyajl.la)
+if ENABLE_COVERAGE
+crun_LDFLAGS = $(CRUN_LDFLAGS) $(COVERAGE_LDFLAGS)
+else
 crun_LDFLAGS = $(CRUN_LDFLAGS)
 endif
+endif
 
 EXTRA_DIST = COPYING COPYING.libcrun README.md NEWS SECURITY.md rpm/crun.spec autogen.sh \
 	src/libcrun/blake3/blake3_impl.h src/libcrun/blake3/blake3.h \
@@ -187,6 +207,7 @@ TESTS_LDADD = libcrun_testing.la $(FOUND_LIBS) $(maybe_libyajl.la)
 
 tests_init_LDADD =
 tests_init_LDFLAGS = -static-libgcc -all-static
+tests_init_CFLAGS = -g -O2
 tests_init_SOURCES = tests/init.c
 
 tests_tests_libcrun_utils_CFLAGS = -I $(abs_top_builddir)/libocispec/src -I $(abs_top_srcdir)/libocispec/src -I $(abs_top_builddir)/src -I $(abs_top_srcdir)/src
@@ -338,4 +359,105 @@ clang-format:
 shellcheck:
 	shellcheck autogen.sh build-aux/release.sh tests/run_all_tests.sh tests/*/*.sh contrib/*.sh
 
-.PHONY: coverity sync generate-rust-bindings generate-signals.c generate-mount_flags.c clang-format shellcheck
+# Code coverage targets
+if ENABLE_COVERAGE
+
+# Clean coverage data
+coverage-clean:
+	@rm -rf coverage-html coverage.info coverage.xml
+	@find . -name "*.gcda" -delete
+	@find . -name "*.gcno" -delete
+
+# Reset coverage counters
+coverage-reset:
+	@if test -n "$(LCOV)"; then \
+		$(LCOV) --zerocounters --directory .; \
+	fi
+
+# Run tests and collect coverage data
+coverage-check: coverage-reset
+	@echo "Running tests for coverage (single-threaded to avoid race conditions)..."
+	$(MAKE) -j1 check
+	@echo "Collecting coverage data..."
+
+# Generate HTML coverage report (preferred method with lcov)
+coverage-html: coverage-check
+	@if test -n "$(LCOV)"; then \
+		echo "Generating coverage report with lcov..."; \
+		$(LCOV) --capture --directory . --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '/usr/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/libocispec/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/test_*.py*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/init*' --output-file coverage.info; \
+		genhtml coverage.info --output-directory coverage-html; \
+		echo "Coverage report generated in coverage-html/index.html"; \
+	elif test -n "$(GCOVR)"; then \
+		echo "Generating coverage report with gcovr..."; \
+		$(GCOVR) --html --html-details -o coverage.html \
+			--exclude '/usr/.*' --exclude '.*/libocispec/.*' --exclude '.*/tests/test_.*\.py.*' --exclude '.*/tests/init.*'; \
+		echo "Coverage report generated in coverage.html"; \
+	else \
+		echo "Generating coverage report with gcov..."; \
+		mkdir -p coverage-html; \
+		for src in $(libcrun_SOURCES) $(crun_SOURCES); do \
+			if test -f "$${src}.gcno"; then \
+				$(GCOV) -o . $$src || true; \
+			fi; \
+		done; \
+		mv *.gcov coverage-html/ 2>/dev/null || true; \
+		echo "Coverage files generated in coverage-html/"; \
+	fi
+
+# Generate XML coverage report (for CI tools)
+coverage-xml: coverage-check
+	@if test -n "$(GCOVR)"; then \
+		echo "Generating XML coverage report with gcovr..."; \
+		$(GCOVR) --xml -o coverage.xml \
+			--exclude '/usr/.*' --exclude '.*/libocispec/.*' --exclude '.*/tests/test_.*\.py.*' --exclude '.*/tests/init.*'; \
+		echo "Coverage report generated in coverage.xml"; \
+	elif test -n "$(LCOV)"; then \
+		echo "Generating XML coverage report with lcov..."; \
+		$(LCOV) --capture --directory . --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '/usr/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/libocispec/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/test_*.py*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/init*' --output-file coverage.info; \
+		echo "Coverage data collected in coverage.info"; \
+	else \
+		echo "XML coverage requires gcovr or lcov"; \
+		exit 1; \
+	fi
+
+# Generate coverage summary
+coverage-summary: coverage-check
+	@if test -n "$(LCOV)"; then \
+		echo "Coverage summary (lcov):"; \
+		$(LCOV) --capture --directory . --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '/usr/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/libocispec/*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/test_*.py*' --output-file coverage.info; \
+		$(LCOV) --remove coverage.info '*/tests/init*' --output-file coverage.info; \
+		$(LCOV) --summary coverage.info; \
+	elif test -n "$(GCOVR)"; then \
+		echo "Coverage summary (gcovr):"; \
+		$(GCOVR) --exclude '/usr/.*' --exclude '.*/libocispec/.*' --exclude '.*/tests/test_.*\.py.*' --exclude '.*/tests/init.*'; \
+	else \
+		echo "Coverage summary requires lcov or gcovr"; \
+	fi
+
+else
+
+coverage-clean:
+	@echo "Coverage support not enabled. Reconfigure with --enable-coverage"
+
+coverage-reset coverage-check coverage-html coverage-xml coverage-summary:
+	@echo "Coverage support not enabled. Reconfigure with --enable-coverage"
+
+endif
+
+clean-local: coverage-clean
+
+# Coverage targets must not run in parallel due to race conditions in .gcda file writes
+.NOTPARALLEL: coverage-reset coverage-check coverage-html coverage-xml coverage-summary
+
+.PHONY: coverity sync generate-rust-bindings generate-signals.c generate-mount_flags.c clang-format shellcheck coverage-clean coverage-reset coverage-check coverage-html coverage-xml coverage-summary

configure.ac

@@ -366,6 +366,45 @@ if test -z "$GPERF"; then
 	AC_MSG_NOTICE(gperf not found - cannot rebuild signal parser code)
 fi
 
+dnl code coverage
+AC_ARG_ENABLE([coverage],
+	AS_HELP_STRING([--enable-coverage], [Enable code coverage support]),
+	[enable_coverage=$enableval], [enable_coverage=no])
+
+AS_IF([test "x$enable_coverage" = "xyes"], [
+	AC_CHECK_TOOL([GCOV], [gcov])
+	if test -z "$GCOV"; then
+		AC_MSG_ERROR([gcov is required for code coverage])
+	fi
+
+	AC_CHECK_TOOL([LCOV], [lcov])
+	AC_CHECK_TOOL([GCOVR], [gcovr])
+
+	# Choose the best available coverage tool
+	if test -n "$LCOV"; then
+		coverage_tool=lcov
+	elif test -n "$GCOVR"; then
+		coverage_tool=gcovr
+	else
+		coverage_tool=gcov
+	fi
+
+	AC_MSG_NOTICE([Using $coverage_tool for code coverage reporting])
+
+	# Add coverage flags
+	COVERAGE_CFLAGS="--coverage -g -O0 -fno-inline -fno-inline-small-functions -fno-default-inline"
+	COVERAGE_LDFLAGS="--coverage"
+
+	AC_SUBST([COVERAGE_CFLAGS])
+	AC_SUBST([COVERAGE_LDFLAGS])
+	AC_SUBST([GCOV])
+	AC_SUBST([LCOV])
+	AC_SUBST([GCOVR])
+	AC_SUBST([coverage_tool], [$coverage_tool])
+])
+
+AM_CONDITIONAL([ENABLE_COVERAGE], [test "x$enable_coverage" = "xyes"])
+
 AC_SEARCH_LIBS([argp_parse], [argp], [], [AC_MSG_ERROR([*** argp functions not found - install libargp or argp_standalone])])
 
 AM_CONDITIONAL([PYTHON_BINDINGS], [test "x$with_python_bindings" = "xyes"])

maint.mk

@@ -1581,7 +1581,7 @@ init-coverage:
 	lcov --directory . --zerocounters
 
 COVERAGE_CCOPTS ?= "-g --coverage"
-COVERAGE_OUT ?= doc/coverage
+COVERAGE_OUT ?= docs/coverage
 
 build-coverage:
 	$(MAKE) $(AM_MAKEFLAGS) CFLAGS=$(COVERAGE_CCOPTS) CXXFLAGS=$(COVERAGE_CCOPTS)
@@ -1594,7 +1594,10 @@ gen-coverage:
 	genhtml --output-directory $(COVERAGE_OUT) \
 		$(COVERAGE_OUT)/$(PACKAGE).info \
 		--highlight --frames --legend \
-		--title "$(PACKAGE_NAME)"
+		--title "$(PACKAGE_NAME)" \
+		--ignore-errors unmapped
+
+.NOTPARALLEL: coverage init-coverage build-coverage gen-coverage
 
 coverage:
 	$(MAKE) init-coverage

tests/test_bpf_devices.py

@@ -47,27 +47,27 @@ def check_bpf_prerequisites():
     """Check all prerequisites for BPF device tests. Returns 77 (skip) if not met, 0 if OK"""
     # Skip if not root
     if is_rootless():
-        return 77
+        return (77, "requires root privileges")
 
     # Skip if not cgroup v2
     if not is_cgroup_v2_unified():
-        return 77
+        return (77, "requires cgroup v2")
 
     # Skip if systemd not available
     if 'SYSTEMD' not in get_crun_feature_string():
-        return 77
+        return (77, "systemd support not compiled in")
 
     # Skip if not running on systemd
     if not running_on_systemd():
-        return 77
+        return (77, "not running on systemd")
 
     # Skip if no BPF support
     if not has_bpf_fs():
-        return 77
+        return (77, "BPF filesystem not available")
 
     # Skip if systemd doesn't support BPFProgram
     if not systemd_supports_bpf_program():
-        return 77
+        return (77, "systemd BPFProgram not supported")
 
     return 0
 
@@ -86,7 +86,7 @@ def test_bpf_devices_systemd():
     bpf_path = None
     try:
         # Run container with systemd cgroup manager.
-        _, cid = run_and_get_output(conf, command='run', detach=True, cgroup_manager="systemd")
+        _, cid = run_and_get_output(conf, hide_stderr=True, command='run', detach=True, cgroup_manager="systemd")
 
         # Get systemd scope.
         state = run_crun_command(['state', cid])
@@ -96,20 +96,20 @@ def test_bpf_devices_systemd():
 
         output = subprocess.check_output(['systemctl', 'show', '-PBPFProgram', scope], close_fds=False).decode().strip()
         if output == "":
-            sys.stderr.write("# BPFProgram property not found or empty\n")
+            logger.info("BPFProgram property not found or empty")
             return -1
 
         # Should look like "device:/sys/fs/bpf/crun/crun-xxx_scope".
         if "device:/sys/fs/bpf/crun/" not in output:
-            sys.stderr.write("# Bad BPFProgram property value: `%s`\n" % output)
+            logger.info("Bad BPFProgram property value: `%s`", prop_value)
             return -1
 
         # Test 2: Check that BPF program file was created.
 
         # Extract the path.
         bpf_path = output.split("device:", 1)[1]
         if not os.path.exists(bpf_path):
-            sys.stderr.write("# BPF program file `%s` not found\n" % bpf_path)
+            logger.info("BPF program file `%s` not found", prog_file)
             return -1
 
         # Test 3: Check that BPF program is cleaned up.
@@ -118,13 +118,13 @@ def test_bpf_devices_systemd():
         run_crun_command(["delete", "-f", cid])
         cid = None
         if os.path.exists(bpf_path):
-            sys.stderr.write("# BPF program `%s` still exist after crun delete\n" % bpf_path)
+            logger.info("BPF program `%s` still exist after crun delete", prog_file)
             return -1
 
         return 0
 
     except Exception as e:
-        sys.stderr.write("# Test failed with exception: %s\n" % str(e))
+        logger.info("Test failed with exception: %s", e)
         return -1
     finally:
         if cid is not None: