From 43b9852748c01219c00f813e60177f04015c8c24 Mon Sep 17 00:00:00 2001 From: Dmitrii Golovanov Date: Sat, 9 Dec 2023 22:12:21 +0100 Subject: [PATCH] twister: coverage: Add optional JSON report on coverage New Twister option `--coverage-sections` allows to select code coverage reports to compose into an additional JSON file (twister_coverage.json). By default, the new option is disabled. Currently this mode is fully supported for the default gcovr reporting tool only, as it is based on the gcovr JSON reports. For lcov, only the coverage run status is reported. The choices are to select either the coverage summary, or the detailed branch coverage, ztest coverage, or all the coverage data collected. Also the coverage run 'status' and coverage 'tool' values are added with any non-default choice. The 'environment' top object has additional 'gcov_tool' property with a full path to what binary tool has been used. Depending on `--coverage-split` and `--disable-coverage-aggregation` options the coverage data is attached either to its test suite object or/and as the report's top level object with the aggregated summary of the current test plan execution scope. In case of the custom report name, or per-platform report, the report name is composed with the rightmost '_coverage.json' suffix. The code coverage report has similar structure as `twister.json` and compelements it having reduced set of test suite properties: - instead of `testcases` it contains `coverage` object with gcov tool JSON report data embedded there; - other properites are limited to represent only the essential test suite context, thus to allow further data processing consistently and independently from the `twister.json`. Signed-off-by: Dmitrii Golovanov --- scripts/pylib/twister/twisterlib/coverage.py | 2 +- .../pylib/twister/twisterlib/environment.py | 31 ++++++-- scripts/pylib/twister/twisterlib/reports.py | 76 +++++++++++++++++-- scripts/pylib/twister/twisterlib/runner.py | 2 +- .../pylib/twister/twisterlib/twister_main.py | 3 +- 5 files changed, 95 insertions(+), 19 deletions(-) diff --git a/scripts/pylib/twister/twisterlib/coverage.py b/scripts/pylib/twister/twisterlib/coverage.py index 1153d5f87f4065..6955fed24025ea 100644 --- a/scripts/pylib/twister/twisterlib/coverage.py +++ b/scripts/pylib/twister/twisterlib/coverage.py @@ -185,7 +185,7 @@ def generate(self, outdir): else: coverage_completed = False logger.debug(f"All coverage data processed: {coverage_completed}") - return coverage_completed, reports + return coverage_completed, reports, self.gcov_tool class Lcov(CoverageTool): diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index a54e702fe47fee..39051144f13e25 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -358,7 +358,8 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: help="Only run cmake, do not build or run.") coverage_group.add_argument("--enable-coverage", action="store_true", - help="Enable code coverage collection using gcov.") + help="Enable code coverage data collection using gcov. " + "Use --coverage to compose reports from that data.") coverage_group.add_argument("-C", "--coverage", action="store_true", help="Generate coverage reports. Implies " @@ -403,6 +404,18 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: be active (`--coverage-split`) to set at least one of the reporting modes. Default: %(default)s""") + coverage_group.add_argument("--coverage-sections", nargs="+", default=None, + choices=['all','report','ztest','summary'], + help="""Selects code coverage data to include into the `twister_coverage.json` report + as its 'coverage' object with properties either as the sections chosen, + or all of them. + With `--coverage-split` each test suite will have its own 'coverage' object + with these sections as properties. + The aggregated coverage of all executed test suites will be a top-level object. + Each 'coverage' object will also have its execution 'status' property. + Currently this mode is fully supported for the default 'gcovr' tool only. + Default: %(default)s""") + parser.add_argument( "--test-config", action="store", @@ -921,13 +934,15 @@ def parse_arguments( if options.enable_coverage and not options.coverage_platform: options.coverage_platform = options.platform - if not options.coverage and (options.disable_coverage_aggregation or options.coverage_split): - logger.error("Enable coverage reporting to set its aggregation mode.") - sys.exit(1) - - if options.coverage and options.disable_coverage_aggregation and not options.coverage_split: - logger.error("At least one coverage reporting mode should be enabled: " - "either aggregation, or split, or both.") + if options.coverage: + if options.disable_coverage_aggregation and not options.coverage_split: + logger.error("At least one coverage reporting mode should be enabled: " + "either aggregation, or split, or both.") + sys.exit(1) + elif (options.disable_coverage_aggregation or + options.coverage_split or + options.coverage_sections): + logger.error("Enable coverage reporting first to set its mode.") sys.exit(1) if options.coverage_formats: diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index f28fbab991a3b8..75530fafe2324d 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # vim: set syntax=python ts=4 : # -# Copyright (c) 2018 Intel Corporation +# Copyright (c) 2018-2025 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import json @@ -33,11 +33,18 @@ class Reporting: json_filters = { 'twister.json': { - 'deny_suite': ['footprint'] + 'deny_property': ['coverage'], + 'deny_suite': ['footprint', 'coverage'] + }, + 'coverage.json': { + 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable', + 'footprint'] }, 'footprint.json': { 'deny_status': ['FILTER'], - 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable'] + 'deny_property': ['coverage'], + 'deny_suite': ['testcases', 'execution_time', 'recording', 'retries', 'runnable', + 'coverage'] } } @@ -51,7 +58,9 @@ def __init__(self, plan, env) -> None: self.outdir = os.path.abspath(env.options.outdir) self.instance_fail_count = plan.instance_fail_count self.footprint = None - + self.coverage_status = None + self.gcov_tool = self.env.options.gcov_tool + self.coverage = None @staticmethod def process_log(log_file): @@ -456,6 +465,17 @@ def json_report(self, filename, version="NA", platform=None, filters=None): # Init as empty data preparing for filtering properties. suite['footprint'] = {} + if ( + instance.status not in [ + TwisterStatus.NONE, + TwisterStatus.ERROR + ] + and self.env.options.coverage_sections + and instance.coverage_status is not None + ): + # Init as empty data preparing for filtering properties. + suite['coverage'] = {} + # Pass suite properties through the context filters. if filters and 'allow_suite' in filters: suite = {k:v for k,v in suite.items() if k in filters['allow_suite']} @@ -476,13 +496,48 @@ def json_report(self, filename, version="NA", platform=None, filters=None): suite['footprint'][k] = json.load(footprint_json) except FileNotFoundError: logger.error(f"Missing footprint.{k} for '{instance.name}'") - # - # + + if 'coverage' in suite and instance.coverage: + suite['coverage']['status'] = instance.coverage_status + suite['coverage']['tool'] = self.env.options.coverage_tool + if instance.coverage.items: + do_all = 'all' in self.env.options.coverage_sections + for k,v in instance.coverage.items(): + if do_all or k in self.env.options.coverage_sections: + logger.debug(f"Include '{instance.name}' coverage.{k} from '{v}'") + try: + with open(v) as json_file: + suite['coverage'][k] = json.load(json_file) + except FileNotFoundError: + logger.error(f"'{instance.name}' missing coverage.{k} from '{v}'") + else: + logger.warning(f"No coverage data for '{instance.name}'") suites.append(suite) report["testsuites"] = suites - with open(filename, 'w') as json_file: + + # Optional report sections with explicit filters + if (filters and \ + ('deny_property' not in filters or 'coverage' not in filters['deny_property']) and \ + self.env.options.coverage_sections and self.coverage_status is not None): + report['environment']['gcov_tool'] = self.gcov_tool + report['coverage'] = { 'status': self.coverage_status, + 'tool': self.env.options.coverage_tool } + if self.coverage.items: + do_all = 'all' in self.env.options.coverage_sections + for k,v in self.coverage.items(): + if do_all or k in self.env.options.coverage_sections: + logger.debug(f"Include aggregated coverage.{k} from '{v}'") + try: + with open(v) as json_file: + report['coverage'][k] = json.load(json_file) + except FileNotFoundError: + logger.error(f"Missing aggregated coverage.{k} from '{v}'") + else: + logger.warning("No aggregated coverage data collected") + + with open(filename, "w") as json_file: json.dump(report, json_file, indent=4, separators=(',',':')) @@ -769,6 +824,9 @@ def save_reports(self, name, suffix, report_dir, no_update, platform_reports): if self.env.options.footprint_report is not None: self.json_report(filename + "_footprint.json", version=self.env.version, filters=self.json_filters['footprint.json']) + if self.env.options.coverage_sections: + self.json_report(filename + "_coverage.json", version=self.env.version, + filters=self.json_filters['coverage.json']) self.xunit_report(json_file, filename + ".xml", full_report=False) self.xunit_report(json_file, filename + "_report.xml", full_report=True) self.xunit_report_suites(json_file, filename + "_suite_report.xml") @@ -794,3 +852,7 @@ def target_report(self, json_file, outdir, suffix): self.json_report(json_platform_file + "_footprint.json", version=self.env.version, platform=platform.name, filters=self.json_filters['footprint.json']) + if self.env.options.coverage_sections: + self.json_report(json_platform_file + "_coverage.json", + version=self.env.version, platform=platform.name, + filters=self.json_filters['coverage.json']) diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 8c7efa926e3c7e..5ec88aea13ad42 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -1152,7 +1152,7 @@ def process(self, pipeline, done, message, lock, results): elif op == "coverage": try: logger.debug(f"Run coverage for '{self.instance.name}'") - self.instance.coverage_status, self.instance.coverage = \ + self.instance.coverage_status, self.instance.coverage, _ = \ run_coverage_instance(self.options, self.instance) next_op = 'report' additionals = { diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index 2c847e49bb1195..a543d48b27520f 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -214,10 +214,9 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): report.summary(runner.results, options.disable_unrecognized_section_test, duration) - report.coverage_status = True if options.coverage and not options.disable_coverage_aggregation: if not options.build_only: - report.coverage_status, report.coverage = run_coverage(options, tplan) + report.coverage_status, report.coverage, report.gcov_tool = run_coverage(options, tplan) else: logger.info("Skipping coverage report generation due to --build-only.")