twister: coverage: Add optional JSON report on coverage

New Twister option `--coverage-sections` allows to select code coverage
reports to compose into an additional JSON file (twister_coverage.json).
By default, the new option is disabled.

Currently this mode is fully supported for the default gcovr reporting
tool only, as it is based on the gcovr JSON reports.
For lcov, only the coverage run status is reported.

The choices are to select either the coverage summary, or the detailed
branch coverage, ztest coverage, or all the coverage data collected.
Also the coverage run 'status' and coverage 'tool' values are added
with any non-default choice. The 'environment' top object has additional
'gcov_tool' property with a full path to what binary tool has been used.

Depending on `--coverage-split` and `--disable-coverage-aggregation`
options the coverage data is attached either to its test suite object
or/and as the report's top level object with the aggregated summary
of the current test plan execution scope.

In case of the custom report name, or per-platform report, the report
name is composed with the rightmost '_coverage.json' suffix.

The code coverage report has similar structure as `twister.json`
and compelements it having reduced set of test suite properties:
- instead of `testcases` it contains `coverage` object with
  gcov tool JSON report data embedded there;
- other properites are limited to represent only the essential test
  suite context, thus to allow further data processing consistently
  and independently from the `twister.json`.

Signed-off-by: Dmitrii Golovanov <[email protected]>

scripts/pylib/twister/twisterlib/coverage.py

@@ -185,7 +185,7 @@ def generate(self, outdir):
             else:
                 coverage_completed = False
         logger.debug(f"All coverage data processed: {coverage_completed}")
-        return coverage_completed, reports
+        return coverage_completed, reports, self.gcov_tool
 
 
 class Lcov(CoverageTool):

scripts/pylib/twister/twisterlib/environment.py

@@ -358,7 +358,8 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser:
         help="Only run cmake, do not build or run.")
 
     coverage_group.add_argument("--enable-coverage", action="store_true",
-                     help="Enable code coverage collection using gcov.")
+                     help="Enable code coverage data collection using gcov. "
+                          "Use --coverage to compose reports from that data.")
 
     coverage_group.add_argument("-C", "--coverage", action="store_true",
                      help="Generate coverage reports. Implies "
@@ -403,6 +404,18 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser:
                         be active (`--coverage-split`) to set at least one of the reporting modes.
                         Default: %(default)s""")
 
+    coverage_group.add_argument("--coverage-sections", nargs="+", default=None,
+            choices=['all','report','ztest','summary'],
+            help="""Selects code coverage data to include into the `twister_coverage.json` report
+                    as its 'coverage' object with properties either as the sections chosen,
+                    or all of them.
+                    With `--coverage-split` each test suite will have its own 'coverage' object
+                    with these sections as properties.
+                    The aggregated coverage of all executed test suites will be a top-level object.
+                    Each 'coverage' object will also have its execution 'status' property.
+                    Currently this mode is fully supported for the default 'gcovr' tool only.
+                    Default: %(default)s""")
+
     parser.add_argument(
         "--test-config",
         action="store",
@@ -921,13 +934,15 @@ def parse_arguments(
     if options.enable_coverage and not options.coverage_platform:
         options.coverage_platform = options.platform
 
-    if not options.coverage and (options.disable_coverage_aggregation or options.coverage_split):
-        logger.error("Enable coverage reporting to set its aggregation mode.")
-        sys.exit(1)
-
-    if options.coverage and options.disable_coverage_aggregation and not options.coverage_split:
-        logger.error("At least one coverage reporting mode should be enabled: "
-                     "either aggregation, or split, or both.")
+    if options.coverage:
+        if options.disable_coverage_aggregation and not options.coverage_split:
+            logger.error("At least one coverage reporting mode should be enabled: "
+                         "either aggregation, or split, or both.")
+            sys.exit(1)
+    elif (options.disable_coverage_aggregation or
+          options.coverage_split or
+          options.coverage_sections):
+        logger.error("Enable coverage reporting first to set its mode.")
         sys.exit(1)
 
     if options.coverage_formats:

scripts/pylib/twister/twisterlib/reports.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: set syntax=python ts=4 :
 #
-# Copyright (c) 2018 Intel Corporation
+# Copyright (c) 2018-2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -33,11 +33,18 @@ class Reporting:
 
     json_filters = {
         'twister.json': {
-            'deny_suite': ['footprint']
+            'deny_property': ['coverage'],
+            'deny_suite': ['footprint', 'coverage']
+        },
+        'coverage.json': {
+            'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable',
+                           'footprint']
         },
         'footprint.json': {
             'deny_status': ['FILTER'],
-            'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable']
+            'deny_property': ['coverage'],
+            'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable',
+                           'coverage']
         }
     }
 
@@ -51,7 +58,9 @@ def __init__(self, plan, env) -> None:
         self.outdir = os.path.abspath(env.options.outdir)
         self.instance_fail_count = plan.instance_fail_count
         self.footprint = None
-
+        self.coverage_status = None
+        self.gcov_tool = self.env.options.gcov_tool
+        self.coverage = None
 
     @staticmethod
     def process_log(log_file):
@@ -456,6 +465,17 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
                 # Init as empty data preparing for filtering properties.
                 suite['footprint'] = {}
 
+            if (
+                instance.status not in [
+                    TwisterStatus.NONE,
+                    TwisterStatus.ERROR
+                ]
+                and self.env.options.coverage_sections
+                and instance.coverage_status is not None
+            ):
+                # Init as empty data preparing for filtering properties.
+                suite['coverage'] = {}
+
             # Pass suite properties through the context filters.
             if filters and 'allow_suite' in filters:
                 suite = {k:v for k,v in suite.items() if k in filters['allow_suite']}
@@ -476,13 +496,48 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
                                 suite['footprint'][k] = json.load(footprint_json)
                         except FileNotFoundError:
                             logger.error(f"Missing footprint.{k} for '{instance.name}'")
-                #
-            #
+
+            if 'coverage' in suite and instance.coverage:
+                suite['coverage']['status'] = instance.coverage_status
+                suite['coverage']['tool'] = self.env.options.coverage_tool
+                if instance.coverage.items:
+                    do_all = 'all' in self.env.options.coverage_sections
+                    for k,v in instance.coverage.items():
+                        if do_all or k in self.env.options.coverage_sections:
+                            logger.debug(f"Include '{instance.name}' coverage.{k} from '{v}'")
+                            try:
+                                with open(v) as json_file:
+                                    suite['coverage'][k] = json.load(json_file)
+                            except FileNotFoundError:
+                                logger.error(f"'{instance.name}' missing coverage.{k} from '{v}'")
+                else:
+                    logger.warning(f"No coverage data for '{instance.name}'")
 
             suites.append(suite)
 
         report["testsuites"] = suites
-        with open(filename, 'w') as json_file:
+
+        # Optional report sections with explicit filters
+        if (filters and \
+            ('deny_property' not in filters or 'coverage' not in filters['deny_property']) and \
+            self.env.options.coverage_sections and self.coverage_status is not None):
+            report['environment']['gcov_tool'] = self.gcov_tool
+            report['coverage'] = { 'status': self.coverage_status,
+                                   'tool': self.env.options.coverage_tool }
+            if self.coverage.items:
+                do_all = 'all' in self.env.options.coverage_sections
+                for k,v in self.coverage.items():
+                    if do_all or k in self.env.options.coverage_sections:
+                        logger.debug(f"Include aggregated coverage.{k} from '{v}'")
+                        try:
+                            with open(v) as json_file:
+                                report['coverage'][k] = json.load(json_file)
+                        except FileNotFoundError:
+                            logger.error(f"Missing aggregated coverage.{k} from '{v}'")
+            else:
+                logger.warning("No aggregated coverage data collected")
+
+        with open(filename, "w") as json_file:
             json.dump(report, json_file, indent=4, separators=(',',':'))
 
 
@@ -769,6 +824,9 @@ def save_reports(self, name, suffix, report_dir, no_update, platform_reports):
             if self.env.options.footprint_report is not None:
                 self.json_report(filename + "_footprint.json", version=self.env.version,
                                  filters=self.json_filters['footprint.json'])
+            if self.env.options.coverage_sections:
+                self.json_report(filename + "_coverage.json", version=self.env.version,
+                                 filters=self.json_filters['coverage.json'])
             self.xunit_report(json_file, filename + ".xml", full_report=False)
             self.xunit_report(json_file, filename + "_report.xml", full_report=True)
             self.xunit_report_suites(json_file, filename + "_suite_report.xml")
@@ -794,3 +852,7 @@ def target_report(self, json_file, outdir, suffix):
                 self.json_report(json_platform_file + "_footprint.json",
                                  version=self.env.version, platform=platform.name,
                                  filters=self.json_filters['footprint.json'])
+            if self.env.options.coverage_sections:
+                self.json_report(json_platform_file + "_coverage.json",
+                                 version=self.env.version, platform=platform.name,
+                                 filters=self.json_filters['coverage.json'])

scripts/pylib/twister/twisterlib/runner.py

@@ -1152,7 +1152,7 @@ def process(self, pipeline, done, message, lock, results):
         elif op == "coverage":
             try:
                 logger.debug(f"Run coverage for '{self.instance.name}'")
-                self.instance.coverage_status, self.instance.coverage = \
+                self.instance.coverage_status, self.instance.coverage, _ = \
                         run_coverage_instance(self.options, self.instance)
                 next_op = 'report'
                 additionals = {

scripts/pylib/twister/twisterlib/twister_main.py

@@ -214,10 +214,9 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
 
     report.summary(runner.results, options.disable_unrecognized_section_test, duration)
 
-    report.coverage_status = True
     if options.coverage and not options.disable_coverage_aggregation:
         if not options.build_only:
-            report.coverage_status, report.coverage = run_coverage(options, tplan)
+            report.coverage_status, report.coverage, report.gcov_tool = run_coverage(options, tplan)
         else:
             logger.info("Skipping coverage report generation due to --build-only.")