Adds verbose mode to the test suites
Added a verbose option to the generated test suites which can list the dependencies not met for skipped test cases. Also clarifies internal interfaces between the main_test.function and test code, and fixed a bug on calculating available tests in run-test-suites.pl.
This commit is contained in:
SimonB
parent
098a3b5025
commit
8ca7bc42d0
@ -2,6 +2,8 @@
|
||||
|
||||
# generate_code.pl
|
||||
#
|
||||
# This file is part of mbed TLS (https://tls.mbed.org)
|
||||
#
|
||||
# Copyright (c) 2009-2016, ARM Limited, All Rights Reserved
|
||||
#
|
||||
# Purpose
|
||||
@ -202,7 +204,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
|
||||
if( substr($def, 0, 4) eq "int " )
|
||||
{
|
||||
$param_defs .= " int param$i;\n";
|
||||
$param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( 2 );\n";
|
||||
$param_checks .= " if( verify_int( params[$i], ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
|
||||
push @dispatch_params, "param$i";
|
||||
|
||||
$mapping_regex .= ":([\\d\\w |\\+\\-\\(\\)]+)";
|
||||
@ -211,7 +213,7 @@ while($test_cases =~ /\/\* BEGIN_CASE *([\w:]*) \*\/\n(.*?)\n\/\* END_CASE \*\//
|
||||
elsif( substr($def, 0, 6) eq "char *" )
|
||||
{
|
||||
$param_defs .= " char *param$i = params[$i];\n";
|
||||
$param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( 2 );\n";
|
||||
$param_checks .= " if( verify_string( ¶m$i ) != 0 ) return( DISPATCH_INVALID_TEST_DATA );\n";
|
||||
push @dispatch_params, "param$i";
|
||||
$mapping_regex .= ":[^:\n]+";
|
||||
}
|
||||
@ -248,14 +250,14 @@ $param_defs
|
||||
if( cnt != $param_count )
|
||||
{
|
||||
mbedtls_fprintf( stderr, "\\nIncorrect argument count (%d != %d)\\n", cnt, $param_count );
|
||||
return( 2 );
|
||||
return( DISPATCH_INVALID_TEST_DATA );
|
||||
}
|
||||
|
||||
$param_checks
|
||||
test_suite_$function_name( $call_params );
|
||||
return ( 0 );
|
||||
return ( DISPATCH_TEST_SUCCESS );
|
||||
$function_post_code
|
||||
return ( 3 );
|
||||
return ( DISPATCH_UNSUPPORTED_SUITE );
|
||||
}
|
||||
else
|
||||
END
|
||||
@ -283,9 +285,9 @@ while( my ($key, $value) = each(%case_deps) )
|
||||
if( strcmp( str, "$key" ) == 0 )
|
||||
{
|
||||
#if defined($key)
|
||||
return( 0 );
|
||||
return( DEPENDENCY_SUPPORTED );
|
||||
#else
|
||||
return( 1 );
|
||||
return( DEPENDENCY_NOT_SUPPORTED );
|
||||
#endif
|
||||
}
|
||||
END
|
||||
@ -298,7 +300,7 @@ while( my ($key, $value) = each(%mapping_values) )
|
||||
if( strcmp( str, "$key" ) == 0 )
|
||||
{
|
||||
*value = ( $key );
|
||||
return( 0 );
|
||||
return( KEY_VALUE_MAPPING_FOUND );
|
||||
}
|
||||
END
|
||||
|
||||
@ -315,7 +317,7 @@ END
|
||||
|
||||
$dispatch_code =~ s/^(.+)/ $1/mg;
|
||||
|
||||
$test_main =~ s/TEST_FILENAME/$test_case_data/;
|
||||
$test_main =~ s/TEST_FILENAME/$test_case_data/g;
|
||||
$test_main =~ s/FUNCTION_CODE//;
|
||||
$test_main =~ s/DEP_CHECK_CODE/$dep_check_code/;
|
||||
$test_main =~ s/DISPATCH_FUNCTION/$dispatch_code/;
|
||||
|
||||
@ -2,6 +2,8 @@
|
||||
|
||||
# run-test-suites.pl
|
||||
#
|
||||
# This file is part of mbed TLS (https://tls.mbed.org)
|
||||
#
|
||||
# Copyright (c) 2015-2016, ARM Limited, All Rights Reserved
|
||||
#
|
||||
# Purpose
|
||||
@ -66,7 +68,8 @@ for my $suite (@suites)
|
||||
print "(test cases passed:", $suite_cases_passed,
|
||||
" failed:", $suite_cases_failed,
|
||||
" skipped:", $suite_cases_skipped,
|
||||
" of total:", ( $suite_cases_passed + $suite_cases_failed ),
|
||||
" of total:", ($suite_cases_passed + $suite_cases_failed +
|
||||
$suite_cases_skipped),
|
||||
")\n"
|
||||
}
|
||||
|
||||
|
||||
@ -32,9 +32,18 @@ typedef UINT32 uint32_t;
|
||||
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
/* Global variables */
|
||||
/* Constants */
|
||||
|
||||
static int test_errors = 0;
|
||||
#define DEPENDENCY_SUPPORTED 0
|
||||
#define DEPENDENCY_NOT_SUPPORTED 1
|
||||
|
||||
#define KEY_VALUE_MAPPING_FOUND 0
|
||||
#define KEY_VALUE_MAPPING_NOT_FOUND -1
|
||||
|
||||
#define DISPATCH_TEST_SUCCESS 0
|
||||
#define DISPATCH_TEST_FN_NOT_FOUND 1
|
||||
#define DISPATCH_INVALID_TEST_DATA 2
|
||||
#define DISPATCH_UNSUPPORTED_SUITE 3
|
||||
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
@ -80,6 +89,12 @@ static int test_errors = 0;
|
||||
#endif
|
||||
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
/* Global variables */
|
||||
|
||||
static int test_errors = 0;
|
||||
|
||||
|
||||
/*----------------------------------------------------------------------------*/
|
||||
/* Helper Functions */
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@ int verify_int( char *str, int *value )
|
||||
MAPPING_CODE
|
||||
|
||||
mbedtls_printf( "Expected integer for parameter and got: %s\n", str );
|
||||
return( -1 );
|
||||
return( KEY_VALUE_MAPPING_NOT_FOUND );
|
||||
}
|
||||
|
||||
|
||||
@ -81,7 +81,7 @@ int dep_check( char *str )
|
||||
|
||||
DEP_CHECK_CODE
|
||||
|
||||
return( 1 );
|
||||
return( DEPENDENCY_NOT_SUPPORTED );
|
||||
}
|
||||
|
||||
int dispatch_test(int cnt, char *params[50])
|
||||
@ -91,14 +91,18 @@ int dispatch_test(int cnt, char *params[50])
|
||||
((void) params);
|
||||
|
||||
#if defined(TEST_SUITE_ACTIVE)
|
||||
ret = DISPATCH_TEST_SUCCESS;
|
||||
|
||||
DISPATCH_FUNCTION
|
||||
{
|
||||
mbedtls_fprintf( stdout, "FAILED\nSkipping unknown test function '%s'\n", params[0] );
|
||||
mbedtls_fprintf( stdout,
|
||||
"FAILED\nSkipping unknown test function '%s'\n",
|
||||
params[0] );
|
||||
fflush( stdout );
|
||||
return( 1 );
|
||||
ret = DISPATCH_TEST_FN_NOT_FOUND;
|
||||
}
|
||||
#else
|
||||
return( 3 );
|
||||
ret = DISPATCH_UNSUPPORTED_SUITE;
|
||||
#endif
|
||||
return( ret );
|
||||
}
|
||||
@ -107,6 +111,19 @@ DISPATCH_FUNCTION
|
||||
/*----------------------------------------------------------------------------*/
|
||||
/* Main Test code */
|
||||
|
||||
#define USAGE \
|
||||
"Usage: %s [OPTIONS] files...\n\n" \
|
||||
" Command line arguments:\n" \
|
||||
" files... One or more test data file. If no file is specified\n" \
|
||||
" the followimg default test case is used:\n" \
|
||||
" %s\n\n" \
|
||||
" Options:\n" \
|
||||
" -v | --verbose Display full information about each test\n" \
|
||||
" -h | --help Display this information\n\n", \
|
||||
argv[0], \
|
||||
"TEST_FILENAME"
|
||||
|
||||
|
||||
int get_line( FILE *f, char *buf, size_t len )
|
||||
{
|
||||
char *ret;
|
||||
@ -216,11 +233,18 @@ static int run_test_snprintf( void )
|
||||
|
||||
int main(int argc, const char *argv[])
|
||||
{
|
||||
int testfile_index, testfile_count, ret, i, cnt;
|
||||
int total_errors = 0, total_tests = 0, total_skipped = 0;
|
||||
/* Local Configurations and options */
|
||||
const char *default_filename = "TEST_FILENAME";
|
||||
const char *test_filename = NULL;
|
||||
const char **test_files = NULL;
|
||||
int testfile_count = 0;
|
||||
int option_verbose = 0;
|
||||
|
||||
/* Other Local variables */
|
||||
int arg_index = 1;
|
||||
const char *next_arg;
|
||||
int testfile_index, ret, i, cnt;
|
||||
int total_errors = 0, total_tests = 0, total_skipped = 0;
|
||||
FILE *file;
|
||||
char buf[5000];
|
||||
char *params[50];
|
||||
@ -253,17 +277,41 @@ int main(int argc, const char *argv[])
|
||||
return( 0 );
|
||||
}
|
||||
|
||||
if ( argc <= 1 )
|
||||
while( arg_index < argc)
|
||||
{
|
||||
next_arg = argv[ arg_index ];
|
||||
|
||||
if( strcmp(next_arg, "--verbose" ) == 0 ||
|
||||
strcmp(next_arg, "-v" ) == 0 )
|
||||
{
|
||||
option_verbose = 1;
|
||||
}
|
||||
else if( strcmp(next_arg, "--help" ) == 0 ||
|
||||
strcmp(next_arg, "-h" ) == 0 )
|
||||
{
|
||||
mbedtls_fprintf( stdout, USAGE );
|
||||
mbedtls_exit( EXIT_SUCCESS );
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Not an option, therefore treat all further arguments as the file
|
||||
* list.
|
||||
*/
|
||||
test_files = &argv[ arg_index ];
|
||||
testfile_count = argc - arg_index;
|
||||
}
|
||||
|
||||
arg_index++;
|
||||
}
|
||||
|
||||
/* If no files were specified, assume a default */
|
||||
if ( test_files == NULL || testfile_count == 0 )
|
||||
{
|
||||
test_files = &default_filename;
|
||||
testfile_count = 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
test_files = &argv[1];
|
||||
testfile_count = argc - 1;
|
||||
}
|
||||
|
||||
/* Now begin to execute the tests in the testfiles */
|
||||
for ( testfile_index = 0;
|
||||
testfile_index < testfile_count;
|
||||
testfile_index++ )
|
||||
@ -280,7 +328,8 @@ int main(int argc, const char *argv[])
|
||||
|
||||
while( !feof( file ) )
|
||||
{
|
||||
int skip = 0;
|
||||
int unmet_dep_count = 0;
|
||||
char *unmet_dependencies[20];
|
||||
|
||||
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
|
||||
break;
|
||||
@ -300,32 +349,61 @@ int main(int argc, const char *argv[])
|
||||
if( strcmp( params[0], "depends_on" ) == 0 )
|
||||
{
|
||||
for( i = 1; i < cnt; i++ )
|
||||
if( dep_check( params[i] ) != 0 )
|
||||
skip = 1;
|
||||
{
|
||||
if( dep_check( params[i] ) != DEPENDENCY_SUPPORTED )
|
||||
{
|
||||
unmet_dependencies[ i-1 ] = strdup(params[i]);
|
||||
if( unmet_dependencies[ i-1 ] == NULL )
|
||||
{
|
||||
mbedtls_printf("FATAL: Out of memory\n");
|
||||
mbedtls_exit( MBEDTLS_PLATFORM_STD_EXIT_FAILURE );
|
||||
}
|
||||
unmet_dep_count++;
|
||||
}
|
||||
}
|
||||
|
||||
if( ( ret = get_line( file, buf, sizeof(buf) ) ) != 0 )
|
||||
break;
|
||||
cnt = parse_arguments( buf, strlen(buf), params );
|
||||
}
|
||||
|
||||
if( skip == 0 )
|
||||
|
||||
// If there are no unmet dependencies execute the test
|
||||
if( unmet_dep_count == 0 )
|
||||
{
|
||||
test_errors = 0;
|
||||
ret = dispatch_test( cnt, params );
|
||||
}
|
||||
|
||||
if( skip == 1 || ret == 3 )
|
||||
if( unmet_dep_count > 0 || ret == DISPATCH_UNSUPPORTED_SUITE )
|
||||
{
|
||||
total_skipped++;
|
||||
mbedtls_fprintf( stdout, "----\n" );
|
||||
|
||||
if( 1 == option_verbose && ret == DISPATCH_UNSUPPORTED_SUITE )
|
||||
{
|
||||
mbedtls_fprintf( stdout, " Test Suite not enabled" );
|
||||
}
|
||||
|
||||
if( 1 == option_verbose && unmet_dep_count > 0 )
|
||||
{
|
||||
mbedtls_fprintf( stdout, " Unmet dependencies: " );
|
||||
while( unmet_dep_count > 0)
|
||||
{
|
||||
mbedtls_fprintf(stdout, "%s ",
|
||||
unmet_dependencies[unmet_dep_count - 1]);
|
||||
free(unmet_dependencies[unmet_dep_count - 1]);
|
||||
unmet_dep_count--;
|
||||
}
|
||||
mbedtls_fprintf( stdout, "\n" );
|
||||
}
|
||||
fflush( stdout );
|
||||
}
|
||||
else if( ret == 0 && test_errors == 0 )
|
||||
else if( ret == DISPATCH_TEST_SUCCESS && test_errors == 0 )
|
||||
{
|
||||
mbedtls_fprintf( stdout, "PASS\n" );
|
||||
fflush( stdout );
|
||||
}
|
||||
else if( ret == 2 )
|
||||
else if( ret == DISPATCH_INVALID_TEST_DATA )
|
||||
{
|
||||
mbedtls_fprintf( stderr, "FAILED: FATAL PARSE ERROR\n" );
|
||||
fclose(file);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user