diff --git a/tests/scripts/generate_code.pl b/tests/scripts/generate_code.pl index 5c623f8a7..5892f7ba3 100755 --- a/tests/scripts/generate_code.pl +++ b/tests/scripts/generate_code.pl @@ -2,6 +2,8 @@ # generate_code.pl # +# This file is part of mbed TLS (https://tls.mbed.org) +# # Copyright (c) 2009-2016, ARM Limited, All Rights Reserved # # Purpose @@ -202,7 +204,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\// if( substr($def, 0, 4) eq "int " ) { $param_defs .= " int param$i;\n"; - $param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( 2 );\n"; + $param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n"; push @dispatch_params, "param$i"; $mapping_regex .= ":([\\d\\w |\\+\\-\\(\\)]+)"; @@ -211,7 +213,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\// elsif( substr($def, 0, 6) eq "char *" ) { $param_defs .= " char *param$i = params[$i];\n"; - $param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( 2 );\n"; + $param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n"; push @dispatch_params, "param$i"; $mapping_regex .= ":[^:\n]+"; } @@ -248,14 +250,14 @@ $param_defs if( cnt != $param_count ) { mbedtls_fprintf( stderr, "\\nIncorrect argument count (%d != %d)\\n", cnt, $param_count ); - return( 2 ); + return( DISPATCH_INVALID_TEST_DATA ); } $param_checks test_suite_$function_name( $call_params ); - return ( 0 ); + return ( DISPATCH_TEST_SUCCESS ); $function_post_code - return ( 3 ); + return ( DISPATCH_UNSUPPORTED_SUITE ); } else END @@ -283,9 +285,9 @@ while( my ($key, $value) = each(%case_deps) ) if( strcmp( str, "$key" ) == 0 ) { #if defined($key) - return( 0 ); + return( DEPENDENCY_SUPPORTED ); #else - return( 1 ); + return( DEPENDENCY_NOT_SUPPORTED ); #endif } END @@ -298,7 +300,7 @@ while( my ($key, $value) = each(%mapping_values) ) if( strcmp( str, "$key" ) == 0 ) { *value = ( $key ); - return( 0 ); + return( KEY_VALUE_MAPPING_FOUND ); } END @@ -315,7 +317,7 @@ END $dispatch_code =~ s/^(.+)/ $1/mg; -$test_main =~ s/TEST_FILENAME/$test_case_data/; +$test_main =~ s/TEST_FILENAME/$test_case_data/g; $test_main =~ s/FUNCTION_CODE//; $test_main =~ s/DEP_CHECK_CODE/$dep_check_code/; $test_main =~ s/DISPATCH_FUNCTION/$dispatch_code/; diff --git a/tests/scripts/run-test-suites.pl b/tests/scripts/run-test-suites.pl index fb77e1571..58f827c14 100755 --- a/tests/scripts/run-test-suites.pl +++ b/tests/scripts/run-test-suites.pl @@ -2,6 +2,8 @@ # run-test-suites.pl # +# This file is part of mbed TLS (https://tls.mbed.org) +# # Copyright (c) 2015-2016, ARM Limited, All Rights Reserved # # Purpose @@ -66,7 +68,8 @@ for my $suite (@suites) print "(test cases passed:", $suite_cases_passed, " failed:", $suite_cases_failed, " skipped:", $suite_cases_skipped, - " of total:", ( $suite_cases_passed + $suite_cases_failed ), + " of total:", ($suite_cases_passed + $suite_cases_failed + + $suite_cases_skipped), ")\n" } diff --git a/tests/suites/helpers.function b/tests/suites/helpers.function index c18eed895..2eff043fd 100644 --- a/tests/suites/helpers.function +++ b/tests/suites/helpers.function @@ -32,9 +32,18 @@ typedef UINT32 uint32_t; /*----------------------------------------------------------------------------*/ -/* Global variables */ +/* Constants */ -static int test_errors = 0; +#define DEPENDENCY_SUPPORTED 0 +#define DEPENDENCY_NOT_SUPPORTED 1 + +#define KEY_VALUE_MAPPING_FOUND 0 +#define KEY_VALUE_MAPPING_NOT_FOUND -1 + +#define DISPATCH_TEST_SUCCESS 0 +#define DISPATCH_TEST_FN_NOT_FOUND 1 +#define DISPATCH_INVALID_TEST_DATA 2 +#define DISPATCH_UNSUPPORTED_SUITE 3 /*----------------------------------------------------------------------------*/ @@ -80,6 +89,12 @@ static int test_errors = 0; #endif +/*----------------------------------------------------------------------------*/ +/* Global variables */ + +static int test_errors = 0; + + /*----------------------------------------------------------------------------*/ /* Helper Functions */ diff --git a/tests/suites/main_test.function b/tests/suites/main_test.function index 7ec69b45d..525df5b24 100644 --- a/tests/suites/main_test.function +++ b/tests/suites/main_test.function @@ -60,7 +60,7 @@ int verify_int( char *str, int *value ) MAPPING_CODE mbedtls_printf( "Expected integer for parameter and got: %s\n", str ); - return( -1 ); + return( KEY_VALUE_MAPPING_NOT_FOUND ); } @@ -81,7 +81,7 @@ int dep_check( char *str ) DEP_CHECK_CODE - return( 1 ); + return( DEPENDENCY_NOT_SUPPORTED ); } int dispatch_test(int cnt, char *params[50]) @@ -91,14 +91,18 @@ int dispatch_test(int cnt, char *params[50]) ((void) params); #if defined(TEST_SUITE_ACTIVE) + ret = DISPATCH_TEST_SUCCESS; + DISPATCH_FUNCTION { - mbedtls_fprintf( stdout, "FAILED\nSkipping unknown test function '%s'\n", params[0] ); + mbedtls_fprintf( stdout, + "FAILED\nSkipping unknown test function '%s'\n", + params[0] ); fflush( stdout ); - return( 1 ); + ret = DISPATCH_TEST_FN_NOT_FOUND; } #else - return( 3 ); + ret = DISPATCH_UNSUPPORTED_SUITE; #endif return( ret ); } @@ -107,6 +111,19 @@ DISPATCH_FUNCTION /*----------------------------------------------------------------------------*/ /* Main Test code */ +#define USAGE \ + "Usage: %s [OPTIONS] files...\n\n" \ + " Command line arguments:\n" \ + " files... One or more test data file. If no file is specified\n" \ + " the followimg default test case is used:\n" \ + " %s\n\n" \ + " Options:\n" \ + " -v | --verbose Display full information about each test\n" \ + " -h | --help Display this information\n\n", \ + argv[0], \ + "TEST_FILENAME" + + int get_line( FILE *f, char *buf, size_t len ) { char *ret; @@ -216,11 +233,18 @@ static int run_test_snprintf( void ) int main(int argc, const char *argv[]) { - int testfile_index, testfile_count, ret, i, cnt; - int total_errors = 0, total_tests = 0, total_skipped = 0; + /* Local Configurations and options */ const char *default_filename = "TEST_FILENAME"; const char *test_filename = NULL; const char **test_files = NULL; + int testfile_count = 0; + int option_verbose = 0; + + /* Other Local variables */ + int arg_index = 1; + const char *next_arg; + int testfile_index, ret, i, cnt; + int total_errors = 0, total_tests = 0, total_skipped = 0; FILE *file; char buf[5000]; char *params[50]; @@ -253,17 +277,41 @@ int main(int argc, const char *argv[]) return( 0 ); } - if ( argc <= 1 ) + while( arg_index < argc) + { + next_arg = argv[ arg_index ]; + + if( strcmp(next_arg, "--verbose" ) == 0 || + strcmp(next_arg, "-v" ) == 0 ) + { + option_verbose = 1; + } + else if( strcmp(next_arg, "--help" ) == 0 || + strcmp(next_arg, "-h" ) == 0 ) + { + mbedtls_fprintf( stdout, USAGE ); + mbedtls_exit( EXIT_SUCCESS ); + } + else + { + /* Not an option, therefore treat all further arguments as the file + * list. + */ + test_files = &argv[ arg_index ]; + testfile_count = argc - arg_index; + } + + arg_index++; + } + + /* If no files were specified, assume a default */ + if ( test_files == NULL || testfile_count == 0 ) { test_files = &default_filename; testfile_count = 1; } - else - { - test_files = &argv[1]; - testfile_count = argc - 1; - } + /* Now begin to execute the tests in the testfiles */ for ( testfile_index = 0; testfile_index < testfile_count; testfile_index++ ) @@ -280,7 +328,8 @@ int main(int argc, const char *argv[]) while( !feof( file ) ) { - int skip = 0; + int unmet_dep_count = 0; + char *unmet_dependencies[20]; if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 ) break; @@ -300,32 +349,61 @@ int main(int argc, const char *argv[]) if( strcmp( params[0], "depends_on" ) == 0 ) { for( i = 1; i < cnt; i++ ) - if( dep_check( params[i] ) != 0 ) - skip = 1; + { + if( dep_check( params[i] ) != DEPENDENCY_SUPPORTED ) + { + unmet_dependencies[ i-1 ] = strdup(params[i]); + if( unmet_dependencies[ i-1 ] == NULL ) + { + mbedtls_printf("FATAL: Out of memory\n"); + mbedtls_exit( MBEDTLS_PLATFORM_STD_EXIT_FAILURE ); + } + unmet_dep_count++; + } + } if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 ) break; cnt = parse_arguments( buf, strlen(buf), params ); } - - if( skip == 0 ) + + // If there are no unmet dependencies execute the test + if( unmet_dep_count == 0 ) { test_errors = 0; ret = dispatch_test( cnt, params ); } - if( skip == 1 || ret == 3 ) + if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE ) { total_skipped++; mbedtls_fprintf( stdout, "----\n" ); + + if( 1 == option_verbose && ret == DISPATCH_UNSUPPORTED_SUITE ) + { + mbedtls_fprintf( stdout, " Test Suite not enabled" ); + } + + if( 1 == option_verbose && unmet_dep_count > 0 ) + { + mbedtls_fprintf( stdout, " Unmet dependencies: " ); + while( unmet_dep_count > 0) + { + mbedtls_fprintf(stdout, "%s ", + unmet_dependencies[unmet_dep_count - 1]); + free(unmet_dependencies[unmet_dep_count - 1]); + unmet_dep_count--; + } + mbedtls_fprintf( stdout, "\n" ); + } fflush( stdout ); } - else if( ret == 0 && test_errors == 0 ) + else if( ret == DISPATCH_TEST_SUCCESS && test_errors == 0 ) { mbedtls_fprintf( stdout, "PASS\n" ); fflush( stdout ); } - else if( ret == 2 ) + else if( ret == DISPATCH_INVALID_TEST_DATA ) { mbedtls_fprintf( stderr, "FAILED: FATAL PARSE ERROR\n" ); fclose(file);