Check test case coverage

Check that every available test case in the test suites and ssl-opt.sh
has been executed at least once.

For the time being, only report a warning, because our coverage is
incomplete. Once we've updated all.sh to have full coverage, this
warning should become an error.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
This commit is contained in:
Gilles Peskine 2020-06-25 18:37:43 +02:00
parent 15c2cbfed5
commit 8d3c70a279

View file

@ -7,9 +7,12 @@ less likely to be useful.
""" """
import argparse import argparse
import re
import sys import sys
import traceback import traceback
import check_test_cases
class Results: class Results:
"""Process analysis results.""" """Process analysis results."""
@ -44,9 +47,40 @@ class TestCaseOutcomes:
""" """
return len(self.successes) + len(self.failures) return len(self.successes) + len(self.failures)
class TestDescriptions(check_test_cases.TestDescriptionExplorer):
"""Collect the available test cases."""
def __init__(self):
super().__init__()
self.descriptions = set()
def process_test_case(self, _per_file_state,
file_name, _line_number, description):
"""Record an available test case."""
base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
key = ';'.join([base_name, description.decode('utf-8')])
self.descriptions.add(key)
def collect_available_test_cases():
"""Collect the available test cases."""
explorer = TestDescriptions()
explorer.walk_all()
return sorted(explorer.descriptions)
def analyze_coverage(results, outcomes):
"""Check that all available test cases are executed at least once."""
available = collect_available_test_cases()
for key in available:
hits = outcomes[key].hits() if key in outcomes else 0
if hits == 0:
# Make this a warning, not an error, as long as we haven't
# fixed this branch to have full coverage of test cases.
results.warning('Test case not executed: {}', key)
def analyze_outcomes(outcomes): def analyze_outcomes(outcomes):
"""Run all analyses on the given outcome collection.""" """Run all analyses on the given outcome collection."""
results = Results() results = Results()
analyze_coverage(results, outcomes)
return results return results
def read_outcome_file(outcome_file): def read_outcome_file(outcome_file):