mbedtls/tests/scripts/run-test-suites.pl

191 lines
6 KiB
Perl
Raw Normal View History

#!/usr/bin/env perl
# run-test-suites.pl
#
# Copyright The Mbed TLS Contributors
Update license headers to Apache-2.0 OR GPL-2.0-or-later This will allow us to ship the LTS branches in a single archive This commit was generated using the following script: # ======================== #!/bin/sh header1='\ * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later\ *\ * This file is provided under the Apache License 2.0, or the\ * GNU General Public License v2.0 or later.\ *\ * **********\ * Apache License 2.0:\ *\ * Licensed under the Apache License, Version 2.0 (the "License"); you may\ * not use this file except in compliance with the License.\ * You may obtain a copy of the License at\ *\ * http://www.apache.org/licenses/LICENSE-2.0\ *\ * Unless required by applicable law or agreed to in writing, software\ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT\ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\ * See the License for the specific language governing permissions and\ * limitations under the License.\ *\ * **********\ *\ * **********\ * GNU General Public License v2.0 or later:\ *\ * This program is free software; you can redistribute it and/or modify\ * it under the terms of the GNU General Public License as published by\ * the Free Software Foundation; either version 2 of the License, or\ * (at your option) any later version.\ *\ * This program is distributed in the hope that it will be useful,\ * but WITHOUT ANY WARRANTY; without even the implied warranty of\ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\ * GNU General Public License for more details.\ *\ * You should have received a copy of the GNU General Public License along\ * with this program; if not, write to the Free Software Foundation, Inc.,\ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\ *\ * **********' find -path './.git' -prune -o '(' -name '*.c' -o -name '*.cpp' -o -name '*.fmt' -o -name '*.h' ')' -print | xargs sed -i " # Normalize the first line of the copyright headers (no text on the first line of a block comment) /^\/\*.*Copyright.*Arm/I s/\/\*/&\n */ # Insert new copyright header /SPDX-License-Identifier/ i\ $header1 # Delete old copyright header /SPDX-License-Identifier/,$ { # Delete lines until the one preceding the mbedtls declaration N 1,/This file is part of/ { /This file is part of/! D } } " # Format copyright header for inclusion into scripts header2=$(echo "$header1" | sed 's/^\\\? \* \?/#/') find -path './.git' -prune -o '(' -name '*.gdb' -o -name '*.pl' -o -name '*.py' -o -name '*.sh' ')' -print | xargs sed -i " # Insert new copyright header /SPDX-License-Identifier/ i\ $header2 # Delete old copyright header /SPDX-License-Identifier/,$ { # Delete lines until the one preceding the mbedtls declaration N 1,/This file is part of/ { /This file is part of/! D } } " # ======================== Signed-off-by: Bence Szépkúti <bence.szepkuti@arm.com>
2020-06-05 11:02:18 +00:00
# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
#
# This file is provided under the Apache License 2.0, or the
# GNU General Public License v2.0 or later.
#
# **********
# Apache License 2.0:
Add Apache-2.0 headers to all scripts This commit was generated using the following script: # ======================== #!/bin/sh # Find scripts find -path './.git' -prune -o '(' -name '*.gdb' -o -name '*.pl' -o -name '*.py' -o -name '*.sh' ')' -print | xargs sed -i ' # Remove Mbed TLS declaration if it occurs before the copyright line 1,/Copyright.*Arm/I { /This file is part of/,$ { /Copyright.*Arm/I! d } } # Convert non-standard header in scripts/abi_check.py to the format used in the other scripts /"""/,/"""/ { # Cut copyright declaration /Copyright.*Arm/I { h N d } # Paste copyright declaration /"""/ { x /./ { s/^/# / # Add # x # Replace orignal buffer with Copyright declaration p # Print original buffer, insert newline i\ s/.*// # Clear original buffer } x } } /Copyright.*Arm/I { # Print copyright declaration p # Read the two lines immediately following the copyright declaration N N # Insert Apache header if it is missing /SPDX/! { i\ # SPDX-License-Identifier: Apache-2.0\ #\ # Licensed under the Apache License, Version 2.0 (the "License"); you may\ # not use this file except in compliance with the License.\ # You may obtain a copy of the License at\ #\ # http://www.apache.org/licenses/LICENSE-2.0\ #\ # Unless required by applicable law or agreed to in writing, software\ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT\ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\ # See the License for the specific language governing permissions and\ # limitations under the License. # Insert Mbed TLS declaration if it is missing /This file is part of/! i\ #\ # This file is part of Mbed TLS (https://tls.mbed.org) } # Clear copyright declaration from buffer D } ' # ======================== Signed-off-by: Bence Szépkúti <bence.szepkuti@arm.com>
2020-05-25 23:54:15 +00:00
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
Update license headers to Apache-2.0 OR GPL-2.0-or-later This will allow us to ship the LTS branches in a single archive This commit was generated using the following script: # ======================== #!/bin/sh header1='\ * SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later\ *\ * This file is provided under the Apache License 2.0, or the\ * GNU General Public License v2.0 or later.\ *\ * **********\ * Apache License 2.0:\ *\ * Licensed under the Apache License, Version 2.0 (the "License"); you may\ * not use this file except in compliance with the License.\ * You may obtain a copy of the License at\ *\ * http://www.apache.org/licenses/LICENSE-2.0\ *\ * Unless required by applicable law or agreed to in writing, software\ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT\ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\ * See the License for the specific language governing permissions and\ * limitations under the License.\ *\ * **********\ *\ * **********\ * GNU General Public License v2.0 or later:\ *\ * This program is free software; you can redistribute it and/or modify\ * it under the terms of the GNU General Public License as published by\ * the Free Software Foundation; either version 2 of the License, or\ * (at your option) any later version.\ *\ * This program is distributed in the hope that it will be useful,\ * but WITHOUT ANY WARRANTY; without even the implied warranty of\ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\ * GNU General Public License for more details.\ *\ * You should have received a copy of the GNU General Public License along\ * with this program; if not, write to the Free Software Foundation, Inc.,\ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.\ *\ * **********' find -path './.git' -prune -o '(' -name '*.c' -o -name '*.cpp' -o -name '*.fmt' -o -name '*.h' ')' -print | xargs sed -i " # Normalize the first line of the copyright headers (no text on the first line of a block comment) /^\/\*.*Copyright.*Arm/I s/\/\*/&\n */ # Insert new copyright header /SPDX-License-Identifier/ i\ $header1 # Delete old copyright header /SPDX-License-Identifier/,$ { # Delete lines until the one preceding the mbedtls declaration N 1,/This file is part of/ { /This file is part of/! D } } " # Format copyright header for inclusion into scripts header2=$(echo "$header1" | sed 's/^\\\? \* \?/#/') find -path './.git' -prune -o '(' -name '*.gdb' -o -name '*.pl' -o -name '*.py' -o -name '*.sh' ')' -print | xargs sed -i " # Insert new copyright header /SPDX-License-Identifier/ i\ $header2 # Delete old copyright header /SPDX-License-Identifier/,$ { # Delete lines until the one preceding the mbedtls declaration N 1,/This file is part of/ { /This file is part of/! D } } " # ======================== Signed-off-by: Bence Szépkúti <bence.szepkuti@arm.com>
2020-06-05 11:02:18 +00:00
# **********
#
# **********
# GNU General Public License v2.0 or later:
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# **********
=head1 SYNOPSIS
Execute all the test suites and print a summary of the results.
run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
Options:
-v|--verbose Print detailed failure information.
-v 2|--verbose=2 Print detailed failure information and summary messages.
-v 3|--verbose=3 Print detailed information about every test case.
--skip=SUITE[,SUITE...]
Skip the specified SUITE(s). This option can be used
multiple times.
=cut
use warnings;
use strict;
use utf8;
use open qw(:std utf8);
use Getopt::Long qw(:config auto_help gnu_compat);
use Pod::Usage;
my $verbose = 0;
my @skip_patterns = ();
GetOptions(
'skip=s' => \@skip_patterns,
'verbose|v:1' => \$verbose,
) or die;
# All test suites = executable files, excluding source files, debug
# and profiling information, etc. We can't just grep {! /\./} because
# some of our test cases' base names contain a dot.
my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*';
@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
die "$0: no test suite found\n" unless @suites;
# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
# but not "test_suite_foobar".
my $skip_re =
( '\Atest_suite_(' .
join('|', map {
s/[ ,;]/|/g; # allow any of " ,;|" as separators
s/\./\./g; # "." in the input means ".", not "any character"
$_
} @skip_patterns) .
')(\z|\.)' );
# in case test suites are linked dynamically
$ENV{'LD_LIBRARY_PATH'} = '../library';
$ENV{'DYLD_LIBRARY_PATH'} = '../library';
my $prefix = $^O eq "MSWin32" ? '' : './';
my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed,
$suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
$total_cases_failed, $total_cases_skipped );
my $suites_skipped = 0;
sub pad_print_center {
my( $width, $padchar, $string ) = @_;
my $padlen = ( $width - length( $string ) - 2 ) / 2;
print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n";
}
for my $suite (@suites)
{
print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
if( $suite =~ /$skip_re/o ) {
print "SKIP\n";
++$suites_skipped;
next;
}
my $command = "$prefix$suite";
if( $verbose ) {
$command .= ' -v';
}
my $result = `$command`;
$suite_cases_passed = () = $result =~ /.. PASS/g;
$suite_cases_failed = () = $result =~ /.. FAILED/g;
$suite_cases_skipped = () = $result =~ /.. ----/g;
if( $? == 0 ) {
print "PASS\n";
if( $verbose > 2 ) {
pad_print_center( 72, '-', "Begin $suite" );
print $result;
pad_print_center( 72, '-', "End $suite" );
}
} else {
$failed_suites++;
print "FAIL\n";
if( $verbose ) {
pad_print_center( 72, '-', "Begin $suite" );
print $result;
pad_print_center( 72, '-', "End $suite" );
}
}
my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/;
$total_tests_run += $tests - $skipped;
if( $verbose > 1 ) {
print "(test cases passed:", $suite_cases_passed,
" failed:", $suite_cases_failed,
" skipped:", $suite_cases_skipped,
" of total:", ($suite_cases_passed + $suite_cases_failed +
$suite_cases_skipped),
")\n"
}
$total_cases_passed += $suite_cases_passed;
$total_cases_failed += $suite_cases_failed;
$total_cases_skipped += $suite_cases_skipped;
}
print "-" x 72, "\n";
print $failed_suites ? "FAILED" : "PASSED";
printf( " (%d suites, %d tests run%s)\n",
scalar(@suites) - $suites_skipped,
$total_tests_run,
$suites_skipped ? ", $suites_skipped suites skipped" : "" );
if( $verbose > 1 ) {
print " test cases passed :", $total_cases_passed, "\n";
print " failed :", $total_cases_failed, "\n";
print " skipped :", $total_cases_skipped, "\n";
print " of tests executed :", ( $total_cases_passed + $total_cases_failed ),
"\n";
print " of available tests :",
( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
"\n";
if( $suites_skipped != 0 ) {
print "Note: $suites_skipped suites were skipped.\n";
}
}
exit( $failed_suites ? 1 : 0 );