diff --git a/scripts/pylib/twister/twisterlib/coverage.py b/scripts/pylib/twister/twisterlib/coverage.py index 1153d5f87f4065..6955fed24025ea 100644 --- a/scripts/pylib/twister/twisterlib/coverage.py +++ b/scripts/pylib/twister/twisterlib/coverage.py @@ -185,7 +185,7 @@ def generate(self, outdir): else: coverage_completed = False logger.debug(f"All coverage data processed: {coverage_completed}") - return coverage_completed, reports + return coverage_completed, reports, self.gcov_tool class Lcov(CoverageTool): diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index a54e702fe47fee..39051144f13e25 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -358,7 +358,8 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: help="Only run cmake, do not build or run.") coverage_group.add_argument("--enable-coverage", action="store_true", - help="Enable code coverage collection using gcov.") + help="Enable code coverage data collection using gcov. " + "Use --coverage to compose reports from that data.") coverage_group.add_argument("-C", "--coverage", action="store_true", help="Generate coverage reports. Implies " @@ -403,6 +404,18 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: be active (`--coverage-split`) to set at least one of the reporting modes. Default: %(default)s""") + coverage_group.add_argument("--coverage-sections", nargs="+", default=None, + choices=['all','report','ztest','summary'], + help="""Selects code coverage data to include into the `twister_coverage.json` report + as its 'coverage' object with properties either as the sections chosen, + or all of them. + With `--coverage-split` each test suite will have its own 'coverage' object + with these sections as properties. + The aggregated coverage of all executed test suites will be a top-level object. + Each 'coverage' object will also have its execution 'status' property. + Currently this mode is fully supported for the default 'gcovr' tool only. + Default: %(default)s""") + parser.add_argument( "--test-config", action="store", @@ -921,13 +934,15 @@ def parse_arguments( if options.enable_coverage and not options.coverage_platform: options.coverage_platform = options.platform - if not options.coverage and (options.disable_coverage_aggregation or options.coverage_split): - logger.error("Enable coverage reporting to set its aggregation mode.") - sys.exit(1) - - if options.coverage and options.disable_coverage_aggregation and not options.coverage_split: - logger.error("At least one coverage reporting mode should be enabled: " - "either aggregation, or split, or both.") + if options.coverage: + if options.disable_coverage_aggregation and not options.coverage_split: + logger.error("At least one coverage reporting mode should be enabled: " + "either aggregation, or split, or both.") + sys.exit(1) + elif (options.disable_coverage_aggregation or + options.coverage_split or + options.coverage_sections): + logger.error("Enable coverage reporting first to set its mode.") sys.exit(1) if options.coverage_formats: diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index f28fbab991a3b8..75530fafe2324d 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: set syntax=python ts=4 : # -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import json @@ -33,11 +33,18 @@ class Reporting: json_filters = { 'twister.json': { - 'deny_suite': ['footprint'] + 'deny_property': ['coverage'], + 'deny_suite': ['footprint', 'coverage'] + }, + 'coverage.json': { + 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable', + 'footprint'] }, 'footprint.json': { 'deny_status': ['FILTER'], - 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable'] + 'deny_property': ['coverage'], + 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable', + 'coverage'] } } @@ -51,7 +58,9 @@ def __init__(self, plan, env) -> None: self.outdir = os.path.abspath(env.options.outdir) self.instance_fail_count = plan.instance_fail_count self.footprint = None - + self.coverage_status = None + self.gcov_tool = self.env.options.gcov_tool + self.coverage = None @staticmethod def process_log(log_file): @@ -456,6 +465,17 @@ def json_report(self, filename, version="NA", platform=None, filters=None): # Init as empty data preparing for filtering properties. suite['footprint'] = {} + if ( + instance.status not in [ + TwisterStatus.NONE, + TwisterStatus.ERROR + ] + and self.env.options.coverage_sections + and instance.coverage_status is not None + ): + # Init as empty data preparing for filtering properties. + suite['coverage'] = {} + # Pass suite properties through the context filters. if filters and 'allow_suite' in filters: suite = {k:v for k,v in suite.items() if k in filters['allow_suite']} @@ -476,13 +496,48 @@ def json_report(self, filename, version="NA", platform=None, filters=None): suite['footprint'][k] = json.load(footprint_json) except FileNotFoundError: logger.error(f"Missing footprint.{k} for '{instance.name}'") - # - # + + if 'coverage' in suite and instance.coverage: + suite['coverage']['status'] = instance.coverage_status + suite['coverage']['tool'] = self.env.options.coverage_tool + if instance.coverage.items: + do_all = 'all' in self.env.options.coverage_sections + for k,v in instance.coverage.items(): + if do_all or k in self.env.options.coverage_sections: + logger.debug(f"Include '{instance.name}' coverage.{k} from '{v}'") + try: + with open(v) as json_file: + suite['coverage'][k] = json.load(json_file) + except FileNotFoundError: + logger.error(f"'{instance.name}' missing coverage.{k} from '{v}'") + else: + logger.warning(f"No coverage data for '{instance.name}'") suites.append(suite) report["testsuites"] = suites - with open(filename, 'w') as json_file: + + # Optional report sections with explicit filters + if (filters and \ + ('deny_property' not in filters or 'coverage' not in filters['deny_property']) and \ + self.env.options.coverage_sections and self.coverage_status is not None): + report['environment']['gcov_tool'] = self.gcov_tool + report['coverage'] = { 'status': self.coverage_status, + 'tool': self.env.options.coverage_tool } + if self.coverage.items: + do_all = 'all' in self.env.options.coverage_sections + for k,v in self.coverage.items(): + if do_all or k in self.env.options.coverage_sections: + logger.debug(f"Include aggregated coverage.{k} from '{v}'") + try: + with open(v) as json_file: + report['coverage'][k] = json.load(json_file) + except FileNotFoundError: + logger.error(f"Missing aggregated coverage.{k} from '{v}'") + else: + logger.warning("No aggregated coverage data collected") + + with open(filename, "w") as json_file: json.dump(report, json_file, indent=4, separators=(',',':')) @@ -769,6 +824,9 @@ def save_reports(self, name, suffix, report_dir, no_update, platform_reports): if self.env.options.footprint_report is not None: self.json_report(filename + "_footprint.json", version=self.env.version, filters=self.json_filters['footprint.json']) + if self.env.options.coverage_sections: + self.json_report(filename + "_coverage.json", version=self.env.version, + filters=self.json_filters['coverage.json']) self.xunit_report(json_file, filename + ".xml", full_report=False) self.xunit_report(json_file, filename + "_report.xml", full_report=True) self.xunit_report_suites(json_file, filename + "_suite_report.xml") @@ -794,3 +852,7 @@ def target_report(self, json_file, outdir, suffix): self.json_report(json_platform_file + "_footprint.json", version=self.env.version, platform=platform.name, filters=self.json_filters['footprint.json']) + if self.env.options.coverage_sections: + self.json_report(json_platform_file + "_coverage.json", + version=self.env.version, platform=platform.name, + filters=self.json_filters['coverage.json']) diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 8c7efa926e3c7e..5ec88aea13ad42 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1152,7 +1152,7 @@ def process(self, pipeline, done, message, lock, results): elif op == "coverage": try: logger.debug(f"Run coverage for '{self.instance.name}'") - self.instance.coverage_status, self.instance.coverage = \ + self.instance.coverage_status, self.instance.coverage, _ = \ run_coverage_instance(self.options, self.instance) next_op = 'report' additionals = { diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index 2c847e49bb1195..a543d48b27520f 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -214,10 +214,9 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): report.summary(runner.results, options.disable_unrecognized_section_test, duration) - report.coverage_status = True if options.coverage and not options.disable_coverage_aggregation: if not options.build_only: - report.coverage_status, report.coverage = run_coverage(options, tplan) + report.coverage_status, report.coverage, report.gcov_tool = run_coverage(options, tplan) else: logger.info("Skipping coverage report generation due to --build-only.")