From 8d3c70a279917c87d46845fb909032ce5351874d Mon Sep 17 00:00:00 2001
From: Gilles Peskine <Gilles.Peskine@arm.com>
Date: Thu, 25 Jun 2020 18:37:43 +0200
Subject: [PATCH] Check test case coverage

Check that every available test case in the test suites and ssl-opt.sh
has been executed at least once.

For the time being, only report a warning, because our coverage is
incomplete. Once we've updated all.sh to have full coverage, this
warning should become an error.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>
---
 tests/scripts/analyze_outcomes.py | 34 +++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index 9d011db55..96599bd53 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -7,9 +7,12 @@ less likely to be useful.
 """
 
 import argparse
+import re
 import sys
 import traceback
 
+import check_test_cases
+
 class Results:
     """Process analysis results."""
 
@@ -44,9 +47,40 @@ class TestCaseOutcomes:
         """
         return len(self.successes) + len(self.failures)
 
+class TestDescriptions(check_test_cases.TestDescriptionExplorer):
+    """Collect the available test cases."""
+
+    def __init__(self):
+        super().__init__()
+        self.descriptions = set()
+
+    def process_test_case(self, _per_file_state,
+                          file_name, _line_number, description):
+        """Record an available test case."""
+        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+        key = ';'.join([base_name, description.decode('utf-8')])
+        self.descriptions.add(key)
+
+def collect_available_test_cases():
+    """Collect the available test cases."""
+    explorer = TestDescriptions()
+    explorer.walk_all()
+    return sorted(explorer.descriptions)
+
+def analyze_coverage(results, outcomes):
+    """Check that all available test cases are executed at least once."""
+    available = collect_available_test_cases()
+    for key in available:
+        hits = outcomes[key].hits() if key in outcomes else 0
+        if hits == 0:
+            # Make this a warning, not an error, as long as we haven't
+            # fixed this branch to have full coverage of test cases.
+            results.warning('Test case not executed: {}', key)
+
 def analyze_outcomes(outcomes):
     """Run all analyses on the given outcome collection."""
     results = Results()
+    analyze_coverage(results, outcomes)
     return results
 
 def read_outcome_file(outcome_file):