twister: coverage: Data collection and reporting per-test instance

The new `--coverage-split` mode extends Twister code coverage operations
to report coverage statistics on each test instance execution individually
in addition to the default reporting mode which aggregates data to one
report with all the test instances in the current scope of the Twister run.
The split mode allows to identify precisely what amount of code coverage
each test case provides and to analyze its contribution to the overall
test plan's coverage. Each build directory will have its own coverage
report and data files, so the overall disk space and the total execution
time increase.
The split mode reads the raw data fies while the tests are running
in parallel, so the report aggregation stage doesn't need to do the same
at its turn.

Another new `--disable-coverage-aggregation` option allows to execute
only the `--coverage-split` mode if the summary statistics are not needed.

Signed-off-by: Dmitrii Golovanov <[email protected]>

scripts/pylib/twister/twisterlib/coverage.py

@@ -1,6 +1,6 @@
 # vim: set syntax=python ts=4 :
 #
-# Copyright (c) 2018-2022 Intel Corporation
+# Copyright (c) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import sys
@@ -29,6 +29,7 @@ def __init__(self):
         self.gcov_tool = None
         self.base_dir = None
         self.output_formats = None
+        self.capture = True
 
     @staticmethod
     def factory(tool):
@@ -102,7 +103,7 @@ def create_gcda_files(extracted_coverage_info):
                 gcda_created = False
         return gcda_created
 
-    def generate(self, outdir):
+    def capture_data(self, outdir):
         coverage_completed = True
         for filename in glob.glob("%s/**/handler.log" % outdir, recursive=True):
             gcov_data = self.__class__.retrieve_gcov_data(filename)
@@ -118,9 +119,13 @@ def generate(self, outdir):
             else:
                 logger.error("Gcov data capture incomplete: {}".format(filename))
                 coverage_completed = False
+        return coverage_completed
 
+    def generate(self, outdir):
+        coverage_completed = self.capture_data(outdir) if self.capture else True
+        reports = {}
         with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
-            ret = self._generate(outdir, coveragelog)
+            ret, reports = self._generate(outdir, coveragelog)
             if ret == 0:
                 report_log = {
                     "html": "HTML report generated: {}".format(os.path.join(outdir, "coverage", "index.html")),
@@ -136,7 +141,7 @@ def generate(self, outdir):
             else:
                 coverage_completed = False
         logger.debug("All coverage data processed: {}".format(coverage_completed))
-        return coverage_completed
+        return coverage_completed, reports
 
 
 class Lcov(CoverageTool):
@@ -206,6 +211,7 @@ def _generate(self, outdir, coveragelog):
         cmd = cmd + ignore_errors
         self.run_command(cmd, coveragelog)
 
+        files = []
         if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
             cmd = ["lcov", "--gcov-tool", self.gcov_tool, "--remove",
                              ztestfile,
@@ -228,7 +234,7 @@ def _generate(self, outdir, coveragelog):
             self.run_command(cmd, coveragelog)
 
         if 'html' not in self.output_formats.split(','):
-            return 0
+            return 0, {}
 
         # The --ignore-errors source option is added to avoid it exiting due to
         # samples/application_development/external_lib/
@@ -237,7 +243,10 @@ def _generate(self, outdir, coveragelog):
                                 "-output-directory",
                                 os.path.join(outdir, "coverage")] + files
         cmd = cmd + ignore_errors
-        return self.run_command(cmd, coveragelog)
+        ret = self.run_command(cmd, coveragelog)
+
+        # TODO: Add support to include LCOV .info files into JSON data.
+        return ret, {}
 
 
 class Gcovr(CoverageTool):
@@ -281,8 +290,9 @@ def _flatten_list(list):
         return [a for b in list for a in b]
 
     def _generate(self, outdir, coveragelog):
-        coveragefile = os.path.join(outdir, "coverage.json")
-        ztestfile = os.path.join(outdir, "ztest.json")
+        coverage_file = os.path.join(outdir, "coverage.json")
+        coverage_summary = os.path.join(outdir, "coverage_summary.json")
+        ztest_file = os.path.join(outdir, "ztest.json")
 
         excludes = Gcovr._interleave_list("-e", self.ignores)
 
@@ -291,24 +301,37 @@ def _generate(self, outdir, coveragelog):
         mode_options = ["--merge-mode-functions=separate"]
 
         # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
-        cmd = ["gcovr", "-r", self.base_dir,
+        cmd = ["gcovr", "-v", "-r", self.base_dir,
                "--gcov-ignore-parse-errors=negative_hits.warn_once_per_file",
                "--gcov-executable", self.gcov_tool,
                "-e", "tests/*"]
-        cmd += excludes + mode_options + ["--json", "-o", coveragefile, outdir]
+        cmd += excludes + mode_options + ["--json", "-o", coverage_file, outdir]
         cmd_str = " ".join(cmd)
-        logger.debug(f"Running {cmd_str}...")
-        subprocess.call(cmd, stdout=coveragelog)
-
-        subprocess.call(["gcovr", "-r", self.base_dir, "--gcov-executable",
-                         self.gcov_tool, "-f", "tests/ztest", "-e",
-                         "tests/ztest/test/*", "--json", "-o", ztestfile,
-                         outdir] + mode_options, stdout=coveragelog)
-
-        if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
-            files = [coveragefile, ztestfile]
+        logger.debug(f"Running: {cmd_str}")
+        coveragelog.write(f"Running: {cmd_str}\n")
+        coveragelog.flush()
+        ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog)
+        if ret:
+            logger.error(f"GCOVR failed with {ret}")
+            return ret, {}
+
+        cmd = ["gcovr", "-v", "-r", self.base_dir] + mode_options
+        cmd += ["--gcov-executable", self.gcov_tool,
+                "-f", "tests/ztest", "-e", "tests/ztest/test/*",
+                "--json", "-o", ztest_file, outdir]
+        cmd_str = " ".join(cmd)
+        logger.debug(f"Running: {cmd_str}")
+        coveragelog.write(f"Running: {cmd_str}\n")
+        coveragelog.flush()
+        ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog)
+        if ret:
+            logger.error(f"GCOVR ztest stage failed with {ret}")
+            return ret, {}
+
+        if os.path.exists(ztest_file) and os.path.getsize(ztest_file) > 0:
+            files = [coverage_file, ztest_file]
         else:
-            files = [coveragefile]
+            files = [coverage_file]
 
         subdir = os.path.join(outdir, "coverage")
         os.makedirs(subdir, exist_ok=True)
@@ -326,19 +349,21 @@ def _generate(self, outdir, coveragelog):
         }
         gcovr_options = self._flatten_list([report_options[r] for r in self.output_formats.split(',')])
 
-        return subprocess.call(["gcovr", "-r", self.base_dir] + mode_options + gcovr_options + tracefiles,
-                               stdout=coveragelog)
+        cmd = ["gcovr", "-v", "-r", self.base_dir] + mode_options + gcovr_options + tracefiles
+        cmd += ["--json-summary-pretty", "--json-summary", coverage_summary]
+        cmd_str = " ".join(cmd)
+        logger.debug(f"Running: {cmd_str}")
+        coveragelog.write(f"Running: {cmd_str}\n")
+        coveragelog.flush()
+        ret = subprocess.call(cmd, stdout=coveragelog, stderr=coveragelog)
+        if ret:
+            logger.error(f"GCOVR merge report stage failed with {ret}")
 
+        return ret, { 'report': coverage_file, 'ztest': ztest_file, 'summary': coverage_summary }
 
 
-def run_coverage(testplan, options):
-    use_system_gcov = False
+def choose_gcov_tool(options, is_system_gcov):
     gcov_tool = None
-
-    for plat in options.coverage_platform:
-        _plat = testplan.get_platform(plat)
-        if _plat and (_plat.type in {"native", "unit"}):
-            use_system_gcov = True
     if not options.gcov_tool:
         zephyr_sdk_gcov_tool = os.path.join(
             os.environ.get("ZEPHYR_SDK_INSTALL_DIR", default=""),
@@ -355,7 +380,7 @@ def run_coverage(testplan, options):
             except OSError:
                 shutil.copy(llvm_cov, gcov_lnk)
             gcov_tool = gcov_lnk
-        elif use_system_gcov:
+        elif is_system_gcov:
             gcov_tool = "gcov"
         elif os.path.exists(zephyr_sdk_gcov_tool):
             gcov_tool = zephyr_sdk_gcov_tool
@@ -365,16 +390,43 @@ def run_coverage(testplan, options):
     else:
         gcov_tool = str(options.gcov_tool)
 
-    logger.info("Generating coverage files...")
-    logger.info(f"Using gcov tool: {gcov_tool}")
+    return gcov_tool
+
+
+def run_coverage_tool(options, outdir, is_system_gcov, capture):
     coverage_tool = CoverageTool.factory(options.coverage_tool)
-    coverage_tool.gcov_tool = gcov_tool
+
+    coverage_tool.gcov_tool = str(choose_gcov_tool(options, is_system_gcov))
+    logger.info(f"Using gcov tool: {coverage_tool.gcov_tool}")
+
+    coverage_tool.capture = capture
     coverage_tool.base_dir = os.path.abspath(options.coverage_basedir)
     # Apply output format default
     if options.coverage_formats is not None:
         coverage_tool.output_formats = options.coverage_formats
     coverage_tool.add_ignore_file('generated')
     coverage_tool.add_ignore_directory('tests')
     coverage_tool.add_ignore_directory('samples')
-    coverage_completed = coverage_tool.generate(options.outdir)
-    return coverage_completed
+
+    logger.info("Generating coverage files...")
+    return coverage_tool.generate(outdir)
+
+
+def has_system_gcov(platform):
+    return platform and (platform.type in {"native", "unit"})
+
+
+def run_coverage(options, testplan):
+    is_system_gcov = False
+
+    for plat in options.coverage_platform:
+        if has_system_gcov(testplan.get_platform(plat)):
+            is_system_gcov = True
+            break
+
+    return run_coverage_tool(options, options.outdir, is_system_gcov, capture=not options.coverage_split)
+
+
+def run_coverage_instance(options, instance):
+    is_system_gcov = has_system_gcov(instance.platform)
+    return run_coverage_tool(options, instance.build_dir, is_system_gcov, capture=True)

scripts/pylib/twister/twisterlib/environment.py

@@ -323,7 +323,7 @@ def add_parse_arguments(parser = None):
                                 "Default to what was selected with --platform.")
 
     coverage_group.add_argument("--coverage-tool", choices=['lcov', 'gcovr'], default='gcovr',
-                                help="Tool to use to generate coverage report (%(default)s - default).")
+                                help="Tool to use to generate coverage reports (%(default)s - default).")
 
     coverage_group.add_argument("--coverage-formats", action="store", default=None,
                                 help="Output formats to use for generated coverage reports " +
@@ -333,6 +333,18 @@ def add_parse_arguments(parser = None):
                                      " Valid options for 'lcov' tool are: " +
                                      ','.join(supported_coverage_formats['lcov']) + " (html,lcov - default).")
 
+    coverage_group.add_argument("--coverage-split", action="store_true", default=False,
+                                help="""Compose individual coverage reports for each test suite
+                                     when coverage reporting is enabled. Might run in addition to
+                                     the default aggregation mode which produces one coverage report for
+                                     all executed test suites. Default: %(default)s""")
+
+    coverage_group.add_argument("--disable-coverage-aggregation", action="store_true", default=False,
+                                help="""Don't aggregate coverage report statistics for all the test suites
+                                     selected to run with enabled coverage. Requires another reporting mode
+                                     to be active (`--coverage-split`) to set at least one reporting mode.
+                                     Default: %(default)s""")
+
     parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"),
         help="Path to file with plans and test configurations.")
 
@@ -769,6 +781,15 @@ def parse_arguments(parser, args, options = None):
     if options.coverage:
         options.enable_coverage = True
 
+    if not options.coverage and (options.disable_coverage_aggregation or options.coverage_split):
+        logger.error("Enable coverage reporting to set its aggregation mode.")
+        sys.exit(1)
+
+    if options.coverage and options.disable_coverage_aggregation and not options.coverage_split:
+        logger.error("At least one coverage reporting mode should be enabled: "
+                     "either aggregation, or split, or both.")
+        sys.exit(1)
+
     if not options.coverage_platform:
         options.coverage_platform = options.platform
 

scripts/pylib/twister/twisterlib/runner.py

@@ -1,6 +1,6 @@
 # vim: set syntax=python ts=4 :
 #
-# Copyright (c) 20180-2022 Intel Corporation
+# Copyright (c) 2018-2023 Intel Corporation
 # Copyright 2022 NXP
 # SPDX-License-Identifier: Apache-2.0
 
@@ -45,6 +45,7 @@
 from twisterlib.platform import Platform
 from twisterlib.testplan import change_skip_to_error_if_integration
 from twisterlib.harness import HarnessImporter, Pytest
+from twisterlib.coverage import run_coverage_instance
 
 logger = logging.getLogger('twister')
 logger.setLevel(logging.DEBUG)
@@ -674,7 +675,7 @@ def process(self, pipeline, done, message, lock, results):
                 self.instance.handler.thread = None
                 self.instance.handler.duts = None
                 pipeline.put({
-                    "op": "report",
+                    "op": "coverage" if self.options.coverage and self.options.coverage_split else "report",
                     "test": self.instance,
                     "status": self.instance.status,
                     "reason": self.instance.reason
@@ -684,6 +685,12 @@ def process(self, pipeline, done, message, lock, results):
                 logger.error(f"RuntimeError: {e}")
                 traceback.print_exc()
 
+        # Run per-instance code coverage
+        elif op == "coverage":
+            logger.debug(f"Run coverage for '{self.instance.name}'")
+            self.instance.coverage_status, self.instance.coverage = run_coverage_instance(self.options, self.instance)
+            pipeline.put({"op": "report", "test": self.instance})
+
         # Report results and output progress to screen
         elif op == "report":
             with lock:

scripts/pylib/twister/twisterlib/testinstance.py

@@ -49,6 +49,8 @@ def __init__(self, testsuite, platform, outdir):
         self.metrics = dict()
         self.handler = None
         self.recording = None
+        self.coverage = None
+        self.coverage_status = None
         self.outdir = outdir
         self.execution_time = 0
         self.build_time = 0

scripts/pylib/twister/twisterlib/twister_main.py

@@ -200,9 +200,9 @@ def main(options):
     report.summary(runner.results, options.disable_unrecognized_section_test, duration)
 
     coverage_completed = True
-    if options.coverage:
+    if options.coverage and not options.disable_coverage_aggregation:
         if not options.build_only:
-            coverage_completed = run_coverage(tplan, options)
+            report.coverage_status, report.coverage = run_coverage(options, tplan)
         else:
             logger.info("Skipping coverage report generation due to --build-only.")
 

scripts/tests/twister/test_runner.py

@@ -1202,7 +1202,7 @@ def mock_getsize(filename, *args, **kwargs):
         mock.ANY,
         ['run test: dummy instance name',
          'run status: dummy instance name success'],
-        {'op': 'report', 'test': mock.ANY, 'status': 'success', 'reason': 'OK'},
+        {'op': 'coverage', 'test': mock.ANY, 'status': 'success', 'reason': 'OK'},
         'success',
         'OK',
         0,