mirror of
https://github.com/yuzu-emu/mbedtls.git
synced 2024-12-23 09:55:32 +00:00
Check test case coverage
Check that every available test case in the test suites and ssl-opt.sh has been executed at least once. For the time being, only report a warning, because our coverage is incomplete. Once we've updated all.sh to have full coverage, this warning should become an error. Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
This commit is contained in:
parent
15c2cbfed5
commit
8d3c70a279
|
@ -7,9 +7,12 @@ less likely to be useful.
|
|||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import check_test_cases
|
||||
|
||||
class Results:
|
||||
"""Process analysis results."""
|
||||
|
||||
|
@ -44,9 +47,40 @@ class TestCaseOutcomes:
|
|||
"""
|
||||
return len(self.successes) + len(self.failures)
|
||||
|
||||
class TestDescriptions(check_test_cases.TestDescriptionExplorer):
|
||||
"""Collect the available test cases."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.descriptions = set()
|
||||
|
||||
def process_test_case(self, _per_file_state,
|
||||
file_name, _line_number, description):
|
||||
"""Record an available test case."""
|
||||
base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
|
||||
key = ';'.join([base_name, description.decode('utf-8')])
|
||||
self.descriptions.add(key)
|
||||
|
||||
def collect_available_test_cases():
|
||||
"""Collect the available test cases."""
|
||||
explorer = TestDescriptions()
|
||||
explorer.walk_all()
|
||||
return sorted(explorer.descriptions)
|
||||
|
||||
def analyze_coverage(results, outcomes):
|
||||
"""Check that all available test cases are executed at least once."""
|
||||
available = collect_available_test_cases()
|
||||
for key in available:
|
||||
hits = outcomes[key].hits() if key in outcomes else 0
|
||||
if hits == 0:
|
||||
# Make this a warning, not an error, as long as we haven't
|
||||
# fixed this branch to have full coverage of test cases.
|
||||
results.warning('Test case not executed: {}', key)
|
||||
|
||||
def analyze_outcomes(outcomes):
|
||||
"""Run all analyses on the given outcome collection."""
|
||||
results = Results()
|
||||
analyze_coverage(results, outcomes)
|
||||
return results
|
||||
|
||||
def read_outcome_file(outcome_file):
|
||||
|
|
Loading…
Reference in a new issue