diff --git a/scripts/pylib/twister/twisterlib/coverage.py b/scripts/pylib/twister/twisterlib/coverage.py index dd647c60ddc347..da3c6a3eb17439 100644 --- a/scripts/pylib/twister/twisterlib/coverage.py +++ b/scripts/pylib/twister/twisterlib/coverage.py @@ -132,6 +132,30 @@ def create_gcda_files(self, extracted_coverage_info): gcda_created = False return gcda_created + def generate_testsuite(self, testsuite_name, outdir): + coverage_completed = True + for filename in glob.glob(f"{outdir}/**/handler.log", recursive=True): + gcov_data = self.__class__.retrieve_gcov_data(filename) + capture_complete = gcov_data['complete'] + extracted_coverage_info = gcov_data['data'] + if capture_complete: + gcda_created = self.create_gcda_files(extracted_coverage_info) + if gcda_created: + logger.debug(f"Gcov data captured: {filename}") + else: + logger.error(f"Gcov data invalid for: {filename}") + coverage_completed = False + else: + logger.error(f"Gcov data capture incomplete: {filename}") + coverage_completed = False + + with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog: + ret = self._generate(outdir, coveragelog, testsuite_name=testsuite_name) + if ret != 0: + coverage_completed = False + logger.debug(f"Testsuite coverage data processed: {coverage_completed}") + return coverage_completed + def generate(self, outdir): coverage_completed = True for filename in glob.glob(f"{outdir}/**/handler.log", recursive=True): @@ -182,6 +206,9 @@ def generate(self, outdir): logger.debug(f"All coverage data processed: {coverage_completed}") return coverage_completed + def merge_coverage(self, outdir): + logger.error("Merge coverage not implemented for %s", self) + return False class Lcov(CoverageTool): @@ -261,24 +288,34 @@ def run_lcov(self, args, coveragelog): ] + parallel + args return self.run_command(cmd, coveragelog) - def _generate(self, outdir, coveragelog): + def _generate(self, outdir, coveragelog, testsuite_name=None): coveragefile = os.path.join(outdir, "coverage.info") ztestfile = os.path.join(outdir, "ztest.info") cmd = ["--capture", "--directory", outdir, "--output-file", coveragefile] - self.run_lcov(cmd, coveragelog) + if testsuite_name: + invalid_chars = re.compile(r"[^A-Za-z0-9_]") + cmd.append("--test-name") + cmd.append(invalid_chars.sub("_", testsuite_name)) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return ret # We want to remove tests/* and tests/ztest/test/* but save tests/ztest cmd = ["--extract", coveragefile, os.path.join(self.base_dir, "tests", "ztest", "*"), "--output-file", ztestfile] - self.run_lcov(cmd, coveragelog) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return ret if os.path.exists(ztestfile) and os.path.getsize(ztestfile) > 0: cmd = ["--remove", ztestfile, os.path.join(self.base_dir, "tests/ztest/test/*"), "--output-file", ztestfile] - self.run_lcov(cmd, coveragelog) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return ret files = [coveragefile, ztestfile] else: @@ -286,9 +323,11 @@ def _generate(self, outdir, coveragelog): for i in self.ignores: cmd = ["--remove", coveragefile, i, "--output-file", coveragefile] - self.run_lcov(cmd, coveragelog) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return ret - if 'html' not in self.output_formats.split(','): + if 'html' not in self.output_formats.split(',') or testsuite_name is not None: return 0 cmd = ["genhtml", "--legend", "--branch-coverage", @@ -296,6 +335,41 @@ def _generate(self, outdir, coveragelog): "-output-directory", os.path.join(outdir, "coverage")] + files return self.run_command(cmd, coveragelog) + def merge_coverage(self, outdir): + logger.info("Merging coverage files...") + with open(os.path.join(outdir, "coverage.log"), "a") as coveragelog: + coveragefile = os.path.join(outdir, "coverage.info") + ztestfile = os.path.join(outdir, "ztest.info") + + # Merge all the coverage.info files into one + cmd = ["--output-file", coveragefile] + for filename in glob.glob(f"{outdir}/**/coverage.info", recursive=True): + cmd.append("--add-tracefile") + cmd.append(filename) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return False + + # Merge all the ztest.info files into one + cmd = ["--output-file", ztestfile] + for filename in glob.glob(f"{outdir}/**/ztest.info", recursive=True): + cmd.append("--add-tracefile") + cmd.append(filename) + ret = self.run_lcov(cmd, coveragelog) + if ret != 0: + return False + + if 'html' not in self.output_formats.split(','): + return True + + # Generate html report + cmd = ["genhtml", "--legend", "--branch-coverage", + "--prefix", self.base_dir, "--show-details", + "-output-directory", os.path.join(outdir, "coverage"), + coveragefile, ztestfile] + ret = self.run_command(cmd, coveragelog) + return ret == 0 + class Gcovr(CoverageTool): @@ -343,7 +417,7 @@ def _interleave_list(prefix, list): def _flatten_list(list): return [a for b in list for a in b] - def _generate(self, outdir, coveragelog): + def _generate(self, outdir, coveragelog, testsuite_name=None): coveragefile = os.path.join(outdir, "coverage.json") ztestfile = os.path.join(outdir, "ztest.json") @@ -404,6 +478,11 @@ def _generate(self, outdir, coveragelog): def run_coverage(testplan, options): + coverage_tool = make_coverage_tool(testplan, options) + coverage_completed = coverage_tool.generate(options.outdir) + return coverage_completed + +def make_coverage_tool(testplan, options): use_system_gcov = False gcov_tool = None @@ -456,5 +535,4 @@ def run_coverage(testplan, options): # Ignore branch coverage on __ASSERT* macros # Covering the failing case is not desirable as it will immediately terminate the test. coverage_tool.add_ignore_branch_pattern(r"^\s*__ASSERT(?:_EVAL|_NO_MSG|_POST_ACTION)?\(.*") - coverage_completed = coverage_tool.generate(options.outdir) - return coverage_completed + return coverage_tool diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index e64fb6934460ad..3e3d3bfc9ac08b 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -345,6 +345,9 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: parser.add_argument("-C", "--coverage", action="store_true", help="Generate coverage reports. Implies " "--enable-coverage.") + parser.add_argument("--coverage-by-testsuite", action="store_true", + help=("Generate coverage data per testsuite, then merge. " + "Requires --coverage.")) parser.add_argument( "-c", "--clobber-output", action="store_true", diff --git a/scripts/pylib/twister/twisterlib/runner.py b/scripts/pylib/twister/twisterlib/runner.py index 8f368d202f63de..6368d552aa4c59 100644 --- a/scripts/pylib/twister/twisterlib/runner.py +++ b/scripts/pylib/twister/twisterlib/runner.py @@ -880,7 +880,7 @@ def parse_generated(self, filter_stages=None): class ProjectBuilder(FilterBuilder): - def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs): + def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, coverage_tool, **kwargs): super().__init__( instance.testsuite, instance.platform, @@ -895,6 +895,7 @@ def __init__(self, instance: TestInstance, env: TwisterEnv, jobserver, **kwargs) self.options = env.options self.env = env self.duts = None + self.coverage_tool = coverage_tool @property def trace(self) -> bool: @@ -1131,6 +1132,8 @@ def process(self, pipeline, done, message, lock, results): self.instance.handler.duts = None next_op = 'report' + if self.coverage_tool is not None: + next_op = 'coverage' additionals = { "status": self.instance.status, "reason": self.instance.reason @@ -1146,6 +1149,34 @@ def process(self, pipeline, done, message, lock, results): finally: self._add_to_pipeline(pipeline, next_op, additionals) + # Run the generated binary using one of the supported handlers + elif op == "coverage": + try: + logger.debug(f"collect coverage: {self.instance.name}") + coverage_ok = self.coverage_tool.generate_testsuite( + self.instance.name, self.instance.build_dir) + + if not coverage_ok: + self.instance.status = TwisterStatus.ERROR + + # to make it work with pickle + self.instance.handler.thread = None + self.instance.handler.duts = None + + next_op = 'report' + additionals = { + } + except StatusAttributeError as sae: + logger.error(str(sae)) + self.instance.status = TwisterStatus.ERROR + reason = 'Incorrect status assignment' + self.instance.reason = reason + self.instance.add_missing_case_status(TwisterStatus.BLOCK, reason) + next_op = 'report' + additionals = {} + finally: + self._add_to_pipeline(pipeline, next_op, additionals) + # Report results and output progress to screen elif op == "report": try: @@ -1803,6 +1834,10 @@ def __init__(self, instances, suites, env=None) -> None: self.jobs = 1 self.results = None self.jobserver = None + self.coverage_tool = None + + def set_coverage_tool(self, coverage_tool): + self.coverage_tool = coverage_tool def run(self): @@ -1966,7 +2001,8 @@ def pipeline_mgr(self, pipeline, done_queue, lock, results): break else: instance = task['test'] - pb = ProjectBuilder(instance, self.env, self.jobserver) + pb = ProjectBuilder(instance, self.env, + self.jobserver, self.coverage_tool) pb.duts = self.duts pb.process(pipeline, done_queue, task, lock, results) if self.env.options.quit_on_failure and \ @@ -1984,7 +2020,7 @@ def pipeline_mgr(self, pipeline, done_queue, lock, results): break else: instance = task['test'] - pb = ProjectBuilder(instance, self.env, self.jobserver) + pb = ProjectBuilder(instance, self.env, self.jobserver, self.coverage_tool) pb.duts = self.duts pb.process(pipeline, done_queue, task, lock, results) if self.env.options.quit_on_failure and \ diff --git a/scripts/pylib/twister/twisterlib/twister_main.py b/scripts/pylib/twister/twisterlib/twister_main.py index 9d4485c885ea04..406b0ec928bc17 100644 --- a/scripts/pylib/twister/twisterlib/twister_main.py +++ b/scripts/pylib/twister/twisterlib/twister_main.py @@ -12,7 +12,7 @@ import colorama from colorama import Fore -from twisterlib.coverage import run_coverage +from twisterlib.coverage import make_coverage_tool, run_coverage from twisterlib.environment import TwisterEnv from twisterlib.hardwaremap import HardwareMap from twisterlib.package import Artifacts @@ -180,6 +180,10 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): tplan.create_build_dir_links() runner = TwisterRunner(tplan.instances, tplan.testsuites, env) + if options.coverage and not options.build_only and options.coverage_by_testsuite: + coverage_tool = make_coverage_tool(tplan, options) + runner.set_coverage_tool(coverage_tool) + # FIXME: This is a workaround for the fact that the hardware map can be usng # the short name of the platform, while the testplan is using the full name. # @@ -217,7 +221,10 @@ def main(options: argparse.Namespace, default_options: argparse.Namespace): coverage_completed = True if options.coverage: if not options.build_only: - coverage_completed = run_coverage(tplan, options) + if options.coverage_by_testsuite: + coverage_completed = coverage_tool.merge_coverage(options.outdir) + else: + coverage_completed = run_coverage(tplan, options) else: logger.info("Skipping coverage report generation due to --build-only.") diff --git a/scripts/tests/twister/test_runner.py b/scripts/tests/twister/test_runner.py index b5eec17b9dac86..5d71fe29f13b9e 100644 --- a/scripts/tests/twister/test_runner.py +++ b/scripts/tests/twister/test_runner.py @@ -69,7 +69,7 @@ def mocked_jobserver(): @pytest.fixture def project_builder(mocked_instance, mocked_env, mocked_jobserver) -> ProjectBuilder: - project_builder = ProjectBuilder(mocked_instance, mocked_env, mocked_jobserver) + project_builder = ProjectBuilder(mocked_instance, mocked_env, mocked_jobserver, None) return project_builder @@ -800,7 +800,7 @@ def mock_abspath(filename, *args, **kwargs): env_mock = mock.Mock() instance_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) with mock.patch('builtins.open', mock_open), \ mock.patch('os.path.realpath', mock_realpath), \ mock.patch('os.path.abspath', mock_abspath): @@ -860,7 +860,7 @@ def mock_getsize(filename, *args, **kwargs): instance_mock.reason = instance_reason instance_mock.build_dir = 'build_dir' - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) log_info_mock = mock.Mock() @@ -1523,7 +1523,7 @@ def mock_determine_testcases(res): instance_mock.testsuite.harness = 'test' env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.options = mock.Mock() pb.options.coverage = options_coverage pb.options.prep_artifacts_for_testing = options_prep_artifacts @@ -1665,7 +1665,7 @@ def test_projectbuilder_determine_testcases( instance_mock.testsuite.detailed_test_id = detailed_id instance_mock.compose_case_name = mock.Mock(side_effect=iter(added_tcs)) - pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver) + pb = ProjectBuilder(instance_mock, mocked_env, mocked_jobserver, None) with mock.patch('twisterlib.runner.ELFFile', elf_mock), \ mock.patch('builtins.open', mock.mock_open()): @@ -1735,7 +1735,7 @@ def test_projectbuilder_cleanup_artifacts( instance_mock.build_dir = tmpdir env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.options = mock.Mock(runtime_artifact_cleanup=runtime_artifact_cleanup) pb.cleanup_artifacts(additional_keep) @@ -1757,7 +1757,7 @@ def test_projectbuilder_cleanup_device_testing_artifacts( instance_mock.build_dir = build_dir env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb._get_binaries = mock.Mock(return_value=bins) pb.cleanup_artifacts = mock.Mock() pb._sanitize_files = mock.Mock() @@ -1812,7 +1812,7 @@ def mock_get_domains(*args, **kwargs): instance_mock.platform.binaries = platform_binaries env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb._get_binaries_from_runners = mock.Mock(return_value=runner_binaries) bins = pb._get_binaries() @@ -1864,7 +1864,7 @@ def mock_exists(fname): instance_mock.build_dir = os.path.join('build', 'dir') env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) with mock.patch('os.path.exists', mock_exists), \ mock.patch('builtins.open', mock.mock_open()), \ @@ -1882,7 +1882,7 @@ def test_projectbuilder_sanitize_files(mocked_jobserver): instance_mock = mock.Mock() env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb._sanitize_runners_file = mock.Mock() pb._sanitize_zephyr_base_from_files = mock.Mock() @@ -1927,7 +1927,7 @@ def mock_exists(fname): instance_mock.build_dir = '/absolute/path/build_dir' env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) with mock.patch('os.path.exists', mock_exists), \ mock.patch('builtins.open', @@ -1989,7 +1989,7 @@ def mock_open(fname, *args, **kwargs): instance_mock.build_dir = build_dir_path env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) with mock.patch('os.path.exists', mock_exists), \ mock.patch('builtins.open', mock_open), \ @@ -2091,7 +2091,7 @@ def test_projectbuilder_report_out( [skip_mock_tc] env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.options.verbose = verbose pb.options.cmake_only = cmake_only pb.options.seed = 123 @@ -2204,7 +2204,7 @@ def test_projectbuilder_cmake(): instance_mock.build_dir = os.path.join('build', 'dir') env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.build_dir = 'build_dir' pb.testsuite.extra_args = ['some', 'args'] pb.testsuite.extra_conf_files = ['some', 'files1'] @@ -2235,7 +2235,7 @@ def test_projectbuilder_build(mocked_jobserver): instance_mock.testsuite.harness = 'test' env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.build_dir = 'build_dir' pb.run_build = mock.Mock(return_value={'dummy': 'dummy'}) @@ -2337,7 +2337,7 @@ def mock_harness(name): instance_mock.testsuite.harness = harness env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.options.extra_test_args = ['dummy_arg1', 'dummy_arg2'] pb.duts = ['another dut'] pb.options.seed = seed @@ -2391,7 +2391,7 @@ def test_projectbuilder_gather_metrics( instance_mock.metrics = {} env_mock = mock.Mock() - pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver) + pb = ProjectBuilder(instance_mock, env_mock, mocked_jobserver, None) pb.options.enable_size_report = enable_size_report pb.options.create_rom_ram_report = False pb.options.cmake_only = cmake_only