twister: Run lcov on each test

If the flag --coverage-by-testsuite is passed to twister, instead of
running lcov on all test output to collect the coverage data, run lcov
on each test separately and them merge the output files.

This should help with parallelism, as I've noticed that running twister
will cause my machine to go to a very high CPU usage until it gets to
the lcov step, then it goes low for several minutes.

Also, the coverage reports will include test details.

Signed-off-by: Jeremy Bettis <[email protected]>

scripts/pylib/twister/twisterlib/coverage.py

@@ -132,6 +132,30 @@ def create_gcda_files(self, extracted_coverage_info):
                 gcda_created = False
         return gcda_created
 
+    def generate_testsuite(self, testsuite_name, outdir):
+        coverage_completed = True
+        for filename in glob.glob(f"{outdir}/**/handler.log", recursive=True):
+            gcov_data = self.__class__.retrieve_gcov_data(filename)
+            capture_complete = gcov_data['complete']
+            extracted_coverage_info = gcov_data['data']
+            if capture_complete:
+                gcda_created = self.create_gcda_files(extracted_coverage_info)
+                if gcda_created:
+                    logger.debug(f"Gcov data captured: {filename}")
+                else:
+                    logger.error(f"Gcov data invalid for: {filename}")
+                    coverage_completed = False
+            else:
+                logger.error(f"Gcov data capture incomplete: {filename}")
+                coverage_completed = False
+
+        with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
+            ret = self._generate(outdir, coveragelog, testsuite_name=testsuite_name)
+            if ret != 0:
+                coverage_completed = False
+        logger.debug(f"Testsuite coverage data processed: {coverage_completed}")
+        return coverage_completed
+
     def generate(self, outdir):
         coverage_completed = True
         for filename in glob.glob(f"{outdir}/**/handler.log", recursive=True):
@@ -182,6 +206,9 @@ def generate(self, outdir):
         logger.debug(f"All coverage data processed: {coverage_completed}")
         return coverage_completed
 
+    def merge_coverage(self, outdir):
+        logger.error("Merge coverage not implemented for %s", self)
+        return False
 
 class Lcov(CoverageTool):
 
@@ -261,41 +288,88 @@ def run_lcov(self, args, coveragelog):
         ] + parallel + args
         return self.run_command(cmd, coveragelog)
 
-    def _generate(self, outdir, coveragelog):
+    def _generate(self, outdir, coveragelog, testsuite_name=None):
         coveragefile = os.path.join(outdir, "coverage.info")
         ztestfile = os.path.join(outdir, "ztest.info")
 
         cmd = ["--capture", "--directory", outdir, "--output-file", coveragefile]
-        self.run_lcov(cmd, coveragelog)
+        if testsuite_name:
+            invalid_chars = re.compile(r"[^A-Za-z0-9_]")
+            cmd.append("--test-name")
+            cmd.append(invalid_chars.sub("_", testsuite_name))
+        ret = self.run_lcov(cmd, coveragelog)
+        if ret != 0:
+            return ret
 
         # We want to remove tests/* and tests/ztest/test/* but save tests/ztest
         cmd = ["--extract", coveragefile,
                os.path.join(self.base_dir, "tests", "ztest", "*"),
                "--output-file", ztestfile]
-        self.run_lcov(cmd, coveragelog)
+        ret = self.run_lcov(cmd, coveragelog)
+        if ret != 0:
+            return ret
 
         if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0:
             cmd = ["--remove", ztestfile,
                    os.path.join(self.base_dir, "tests/ztest/test/*"),
                    "--output-file", ztestfile]
-            self.run_lcov(cmd, coveragelog)
+            ret = self.run_lcov(cmd, coveragelog)
+            if ret != 0:
+                return ret
 
             files = [coveragefile, ztestfile]
         else:
             files = [coveragefile]
 
         for i in self.ignores:
             cmd = ["--remove", coveragefile, i, "--output-file", coveragefile]
-            self.run_lcov(cmd, coveragelog)
+            ret = self.run_lcov(cmd, coveragelog)
+            if ret != 0:
+                return ret
 
-        if 'html' not in self.output_formats.split(','):
+        if 'html' not in self.output_formats.split(',') or testsuite_name is not None:
             return 0
 
         cmd = ["genhtml", "--legend", "--branch-coverage",
                "--prefix", self.base_dir,
                "-output-directory", os.path.join(outdir, "coverage")] + files
         return self.run_command(cmd, coveragelog)
 
+    def merge_coverage(self, outdir):
+        logger.info("Merging coverage files...")
+        with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog:
+            coveragefile = os.path.join(outdir, "coverage.info")
+            ztestfile = os.path.join(outdir, "ztest.info")
+
+            # Merge all the coverage.info files into one
+            cmd = ["--output-file", coveragefile]
+            for filename in glob.glob(f"{outdir}/**/coverage.info", recursive=True):
+                cmd.append("--add-tracefile")
+                cmd.append(filename)
+            ret = self.run_lcov(cmd, coveragelog)
+            if ret != 0:
+                return False
+
+            # Merge all the ztest.info files into one
+            cmd = ["--output-file", ztestfile]
+            for filename in glob.glob(f"{outdir}/**/ztest.info", recursive=True):
+                cmd.append("--add-tracefile")
+                cmd.append(filename)
+            ret = self.run_lcov(cmd, coveragelog)
+            if ret != 0:
+                return False
+
+            if 'html' not in self.output_formats.split(','):
+                return True
+
+            # Generate html report
+            cmd = ["genhtml", "--legend", "--branch-coverage",
+                "--prefix", self.base_dir, "--show-details",
+                "-output-directory", os.path.join(outdir, "coverage"),
+                coveragefile, ztestfile]
+            ret = self.run_command(cmd, coveragelog)
+            return ret == 0
+
 
 class Gcovr(CoverageTool):
 
@@ -343,7 +417,7 @@ def _interleave_list(prefix, list):
     def _flatten_list(list):
         return [a for b in list for a in b]
 
-    def _generate(self, outdir, coveragelog):
+    def _generate(self, outdir, coveragelog, testsuite_name=None):
         coveragefile = os.path.join(outdir, "coverage.json")
         ztestfile = os.path.join(outdir, "ztest.json")
 
@@ -404,6 +478,11 @@ def _generate(self, outdir, coveragelog):
 
 
 def run_coverage(testplan, options):
+    coverage_tool = make_coverage_tool(testplan, options)
+    coverage_completed = coverage_tool.generate(options.outdir)
+    return coverage_completed
+
+def make_coverage_tool(testplan, options):
     use_system_gcov = False
     gcov_tool = None
 
@@ -456,5 +535,4 @@ def run_coverage(testplan, options):
     # Ignore branch coverage on __ASSERT* macros
     # Covering the failing case is not desirable as it will immediately terminate the test.
     coverage_tool.add_ignore_branch_pattern(r"^\s*__ASSERT(?:_EVAL|_NO_MSG|_POST_ACTION)?\(.*")
-    coverage_completed = coverage_tool.generate(options.outdir)
-    return coverage_completed
+    return coverage_tool

scripts/pylib/twister/twisterlib/environment.py

@@ -345,6 +345,9 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser:
     parser.add_argument("-C", "--coverage", action="store_true",
                         help="Generate coverage reports. Implies "
                              "--enable-coverage.")
+    parser.add_argument("--coverage-by-testsuite", action="store_true",
+                        help=("Generate coverage data per testsuite, then merge. "
+                              "Requires --coverage."))
 
     parser.add_argument(
         "-c", "--clobber-output", action="store_true",

scripts/pylib/twister/twisterlib/runner.py

@@ -880,7 +880,7 @@ def parse_generated(self, filter_stages=None):
 
 class ProjectBuilder(FilterBuilder):
 
-    def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs):
+    def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, coverage_tool, **kwargs):
         super().__init__(
             instance.testsuite,
             instance.platform,
@@ -895,6 +895,7 @@ def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs)
         self.options = env.options
         self.env = env
         self.duts = None
+        self.coverage_tool = coverage_tool
 
     @property
     def trace(self) -> bool:
@@ -1131,6 +1132,8 @@ def process(self, pipeline, done, message, lock, results):
                 self.instance.handler.duts = None
 
                 next_op = 'report'
+                if self.coverage_tool is not None:
+                    next_op = 'coverage'
                 additionals = {
                     "status": self.instance.status,
                     "reason": self.instance.reason
@@ -1146,6 +1149,34 @@ def process(self, pipeline, done, message, lock, results):
             finally:
                 self._add_to_pipeline(pipeline, next_op, additionals)
 
+        # Run the generated binary using one of the supported handlers
+        elif op == "coverage":
+            try:
+                logger.debug(f"collect coverage: {self.instance.name}")
+                coverage_ok = self.coverage_tool.generate_testsuite(
+                    self.instance.name, self.instance.build_dir)
+
+                if not coverage_ok:
+                    self.instance.status = TwisterStatus.ERROR
+
+                # to make it work with pickle
+                self.instance.handler.thread = None
+                self.instance.handler.duts = None
+
+                next_op = 'report'
+                additionals = {
+                }
+            except StatusAttributeError as sae:
+                logger.error(str(sae))
+                self.instance.status = TwisterStatus.ERROR
+                reason = 'Incorrect status assignment'
+                self.instance.reason = reason
+                self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason)
+                next_op = 'report'
+                additionals = {}
+            finally:
+                self._add_to_pipeline(pipeline, next_op, additionals)
+
         # Report results and output progress to screen
         elif op == "report":
             try:
@@ -1803,6 +1834,10 @@ def __init__(self, instances, suites, env=None) -> None:
         self.jobs = 1
         self.results = None
         self.jobserver = None
+        self.coverage_tool = None
+
+    def set_coverage_tool(self, coverage_tool):
+        self.coverage_tool = coverage_tool
 
     def run(self):
 
@@ -1966,7 +2001,8 @@ def pipeline_mgr(self, pipeline, done_queue, lock, results):
                             break
                         else:
                             instance = task['test']
-                            pb = ProjectBuilder(instance, self.env, self.jobserver)
+                            pb = ProjectBuilder(instance, self.env,
+                                self.jobserver, self.coverage_tool)
                             pb.duts = self.duts
                             pb.process(pipeline, done_queue, task, lock, results)
                             if self.env.options.quit_on_failure and \
@@ -1984,7 +2020,7 @@ def pipeline_mgr(self, pipeline, done_queue, lock, results):
                         break
                     else:
                         instance = task['test']
-                        pb = ProjectBuilder(instance, self.env, self.jobserver)
+                        pb = ProjectBuilder(instance, self.env, self.jobserver, self.coverage_tool)
                         pb.duts = self.duts
                         pb.process(pipeline, done_queue, task, lock, results)
                         if self.env.options.quit_on_failure and \

scripts/pylib/twister/twisterlib/twister_main.py

@@ -12,7 +12,7 @@
 
 import colorama
 from colorama import Fore
-from twisterlib.coverage import run_coverage
+from twisterlib.coverage import make_coverage_tool, run_coverage
 from twisterlib.environment import TwisterEnv
 from twisterlib.hardwaremap import HardwareMap
 from twisterlib.package import Artifacts
@@ -180,6 +180,10 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
         tplan.create_build_dir_links()
 
     runner = TwisterRunner(tplan.instances, tplan.testsuites, env)
+    if options.coverage and not options.build_only and options.coverage_by_testsuite:
+        coverage_tool = make_coverage_tool(tplan, options)
+        runner.set_coverage_tool(coverage_tool)
+
     # FIXME: This is a workaround for the fact that the hardware map can be usng
     # the short name of the platform, while the testplan is using the full name.
     #
@@ -217,7 +221,10 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace):
     coverage_completed = True
     if options.coverage:
         if not options.build_only:
-            coverage_completed = run_coverage(tplan, options)
+            if options.coverage_by_testsuite:
+                coverage_completed = coverage_tool.merge_coverage(options.outdir)
+            else:
+                coverage_completed = run_coverage(tplan, options)
         else:
             logger.info("Skipping coverage report generation due to --build-only.")