Merge remote-tracking branch 'origin/pr/2255' into mbedtls-2.7

* origin/pr/2255:
  Add a facility to skip running some test suites
  run-test-suites: update the documentation

tests/CMakeLists.txt

@@ -15,6 +15,13 @@ if(NOT PERL_FOUND)
     message(FATAL_ERROR "Cannot build test suites without Perl")
 endif()
 
+# Test suites caught by SKIP_TEST_SUITES are built but not executed.
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+string(REGEX REPLACE "[ ,;]" "|" SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES}")
+string(REPLACE "." "\\." SKIP_TEST_SUITES_REGEX "${SKIP_TEST_SUITES_REGEX}")
+set(SKIP_TEST_SUITES_REGEX "^(${SKIP_TEST_SUITES_REGEX})(\$|\\.)")
+
 function(add_test_suite suite_name)
     if(ARGV1)
         set(data_name ${ARGV1})
@@ -31,7 +38,11 @@ function(add_test_suite suite_name)
     include_directories(${CMAKE_CURRENT_SOURCE_DIR})
     add_executable(test_suite_${data_name} test_suite_${data_name}.c)
     target_link_libraries(test_suite_${data_name} ${libs})
-    add_test(${data_name}-suite test_suite_${data_name} --verbose)
+    if(${data_name} MATCHES ${SKIP_TEST_SUITES_REGEX})
+        message(STATUS "The test suite ${data_name} will not be executed.")
+    else()
+        add_test(${data_name}-suite test_suite_${data_name} --verbose)
+    endif()
 endfunction(add_test_suite)
 
 if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)

tests/Makefile

@@ -444,7 +444,8 @@ else
 	del /Q /F *.c *.exe
 endif
 
+# Test suites caught by SKIP_TEST_SUITES are built but not executed.
 check: $(APPS)
-	perl scripts/run-test-suites.pl
+	perl scripts/run-test-suites.pl --skip=$(SKIP_TEST_SUITES)
 
 test: check

tests/scripts/run-test-suites.pl

@@ -4,19 +4,24 @@
 #
 # This file is part of mbed TLS (https://tls.mbed.org)
 #
-# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
-#
-# Purpose
-#
-# Executes all the available test suites, and provides a basic summary of the
-# results.
-#
-# Usage: run-test-suites.pl [-v]
-#
-# Options :
-#   -v|--verbose    - Provide a pass/fail/skip breakdown per test suite and
-#                     in total
-#
+# Copyright (c) 2015-2018, ARM Limited, All Rights Reserved
+
+=head1 SYNOPSIS
+
+Execute all the test suites and print a summary of the results.
+
+ run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
+
+Options:
+
+  -v|--verbose        Print detailed failure information.
+  -v 2|--verbose=2    Print detailed failure information and summary messages.
+  -v 3|--verbose=3    Print detailed information about every test case.
+  --skip=SUITE[,SUITE...]
+                      Skip the specified SUITE(s). This option can be used
+                      multiple times.
+
+=cut
 
 use warnings;
 use strict;
@@ -24,10 +29,15 @@ use strict;
 use utf8;
 use open qw(:std utf8);
 
-use Getopt::Long;
+use Getopt::Long qw(:config auto_help gnu_compat);
+use Pod::Usage;
 
 my $verbose = 0;
-GetOptions( "verbose|v:1" => \$verbose );
+my @skip_patterns = ();
+GetOptions(
+           'skip=s' => \@skip_patterns,
+           'verbose|v:1' => \$verbose,
+          ) or die;
 
 # All test suites = executable files, excluding source files, debug
 # and profiling information, etc. We can't just grep {! /\./} because
@@ -36,6 +46,17 @@ my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*';
 @suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
 die "$0: no test suite found\n" unless @suites;
 
+# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
+# but not "test_suite_foobar".
+my $skip_re =
+    ( '\Atest_suite_(' .
+      join('|', map {
+          s/[ ,;]/|/g; # allow any of " ,;|" as separators
+          s/\./\./g; # "." in the input means ".", not "any character"
+          $_
+      } @skip_patterns) .
+      ')(\z|\.)' );
+
 # in case test suites are linked dynamically
 $ENV{'LD_LIBRARY_PATH'} = '../library';
 $ENV{'DYLD_LIBRARY_PATH'} = '../library';
@@ -45,6 +66,7 @@ my $prefix = $^O eq "MSWin32" ? '' : './';
 my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed,
     $suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
     $total_cases_failed, $total_cases_skipped );
+my $suites_skipped = 0;
 
 sub pad_print_center {
     my( $width, $padchar, $string ) = @_;
@@ -55,6 +77,12 @@ sub pad_print_center {
 for my $suite (@suites)
 {
     print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
+    if( $suite =~ /$skip_re/o ) {
+        print "SKIP\n";
+        ++$suites_skipped;
+        next;
+    }
+
     my $command = "$prefix$suite";
     if( $verbose ) {
         $command .= ' -v';
@@ -101,7 +129,10 @@ for my $suite (@suites)
 
 print "-" x 72, "\n";
 print $failed_suites ? "FAILED" : "PASSED";
-printf " (%d suites, %d tests run)\n", scalar @suites, $total_tests_run;
+printf( " (%d suites, %d tests run%s)\n",
+        scalar(@suites) - $suites_skipped,
+        $total_tests_run,
+        $suites_skipped ? ", $suites_skipped suites skipped" : "" );
 
 if( $verbose > 1 ) {
     print "  test cases passed :", $total_cases_passed, "\n";
@@ -111,8 +142,11 @@ if( $verbose > 1 ) {
             "\n";
     print " of available tests :",
             ( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
-            "\n"
+            "\n";
+    if( $suites_skipped != 0 ) {
+        print "Note: $suites_skipped suites were skipped.\n";
     }
+}
 
 exit( $failed_suites ? 1 : 0 );