twister: coverage: Add coverage data to twister.json

New Twister option `--coverage-sections` allow to select what coverage
data to include into the `twister.json` file for its further analysis
in addition to the test meta-data and the test results.

The choices are to select either the coverage summary, or the detailed
branch coverage, ztest coverage, or all the coverage data collected.
Also the coverage run 'status' and coverage 'tool' values are added
with any non-default choice. The 'environment' top object has additional
'gcov_tool' proverty to keep the command line argument selection.

Depending on `--coverage-split` and `--disable-coverage-aggregation`
options the coverage data is attached either to its test suite object
or/and at the report's top level as the aggregated summary of the test
plan execution scope.

Currently this mode is fully supported for the default gcovr reporting
tool only, as it is based on the GCOVR json reports. For lcov only
coverage run status is reported.

Signed-off-by: Dmitrii Golovanov <[email protected]>

scripts/pylib/twister/twisterlib/environment.py

@@ -345,6 +345,17 @@ def add_parse_arguments(parser = None):
                                      to be active (`--coverage-split`) to set at least one reporting mode.
                                      Default: %(default)s""")
 
+    coverage_group.add_argument("--coverage-sections", nargs="+", default=None,
+                        choices=['all','report','ztest','summary'],
+                        help="""Selects coverage data to include into the `twister.json` report as 'coverage'
+                                object and its properties either as the sections chosen, or all of them.
+                                With `--coverage-split` each test suite will have its own 'coverage' object
+                                with these sections as properties.
+                                The aggregated coverage of all executed test suites will be a top-level object.
+                                Each 'coverage' object will also have its execution 'status' property.
+                                Currently this mode is fully supported for the default 'gcovr' tool only.
+                                Default: %(default)s""")
+
     parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"),
         help="Path to file with plans and test configurations.")
 

scripts/pylib/twister/twisterlib/reports.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: set syntax=python ts=4 :
 #
-# Copyright (c) 2018 Intel Corporation
+# Copyright (c) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -26,6 +26,8 @@ def __init__(self, plan, env) -> None:
         self.env = env
         self.timestamp = datetime.now().isoformat()
         self.outdir = os.path.abspath(env.options.outdir)
+        self.coverage_status = None
+        self.coverage = None
 
     @staticmethod
     def process_log(log_file):
@@ -351,9 +353,31 @@ def json_report(self, filename, version="NA", platform=None):
             if instance.recording is not None:
                 suite['recording'] = instance.recording
 
+            if self.env.options.coverage_sections and instance.coverage_status is not None:
+                suite['coverage'] = { 'status': instance.coverage_status,
+                                      'tool': self.env.options.coverage_tool }
+                do_all = 'all' in self.env.options.coverage_sections
+                for k,v in instance.coverage.items():
+                    if do_all or k in self.env.options.coverage_sections:
+                        logger.debug(f"Include '{instance.name}' coverage.{k} from '{v}'")
+                        with open(v, "rt") as json_file:
+                            suite['coverage'][k] = json.load(json_file)
+
             suites.append(suite)
 
         report["testsuites"] = suites
+
+        if self.env.options.coverage_sections and self.coverage_status is not None:
+            report['environment']['gcov_tool'] = self.env.options.gcov_tool
+            report['coverage'] = { 'status': self.coverage_status,
+                                   'tool': self.env.options.coverage_tool }
+            do_all = 'all' in self.env.options.coverage_sections
+            for k,v in self.coverage.items():
+                if do_all or k in self.env.options.coverage_sections:
+                    logger.debug(f"Include aggregated coverage.{k} from '{v}'")
+                    with open(v, "rt") as json_file:
+                        report['coverage'][k] = json.load(json_file)
+
         with open(filename, "wt") as json_file:
             json.dump(report, json_file, indent=4, separators=(',',':'))