From d025422c288c10243f568548f3b54065c049f513 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 14 Sep 2021 11:28:22 +0200 Subject: [PATCH] Remove on-target testing It was unmaintained and untested, and the fear of breaking it was holding us back. Resolves #4934. Signed-off-by: Gilles Peskine --- ChangeLog.d/remove-greentea-support.txt | 3 + tests/Makefile | 39 +- tests/scripts/generate_test_code.py | 4 - tests/scripts/mbedtls_test.py | 382 -------------------- tests/suites/target_test.function | 449 ------------------------ 5 files changed, 4 insertions(+), 873 deletions(-) create mode 100644 ChangeLog.d/remove-greentea-support.txt delete mode 100755 tests/scripts/mbedtls_test.py delete mode 100644 tests/suites/target_test.function diff --git a/ChangeLog.d/remove-greentea-support.txt b/ChangeLog.d/remove-greentea-support.txt new file mode 100644 index 000000000..af4df4baa --- /dev/null +++ b/ChangeLog.d/remove-greentea-support.txt @@ -0,0 +1,3 @@ +Removals + * Remove the partial support for running unit tests via Greentea on Mbed OS, + which had been unmaintained since 2018. diff --git a/tests/Makefile b/tests/Makefile index db642c779..77a31720f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -173,7 +173,7 @@ $(BINARIES): %$(EXEXT): %.c $(MBEDLIBS) $(TEST_OBJS_DEPS) $(MBEDTLS_TEST_OBJS) clean: ifndef WINDOWS - rm -rf $(BINARIES) *.c *.datax TESTS + rm -rf $(BINARIES) *.c *.datax rm -f src/*.o src/drivers/*.o src/libmbed* rm -f include/test/instrument_record_status.h else @@ -184,9 +184,6 @@ else if exist src/drivers/*.o del /Q /F src/drivers/*.o if exist src/libmbed* del /Q /F src/libmed* if exist include/test/instrument_record_status.h del /Q /F include/test/instrument_record_status.h -ifneq ($(wildcard TESTS/.*),) - rmdir /Q /S TESTS -endif endif neat: clean @@ -202,40 +199,6 @@ check: $(BINARIES) test: check -# Create separate targets for generating embedded tests. -EMBEDDED_TESTS := $(addprefix embedded_,$(APPS)) - -# Generate test code for target. - -.SECONDEXPANSION: -$(EMBEDDED_TESTS): embedded_%: suites/$$(firstword $$(subst ., ,$$*)).function suites/%.data scripts/generate_test_code.py suites/helpers.function suites/main_test.function suites/target_test.function - echo " Gen ./TESTS/mbedtls/$*/$*.c" - $(PYTHON) scripts/generate_test_code.py -f suites/$(firstword $(subst ., ,$*)).function \ - -d suites/$*.data \ - -t suites/main_test.function \ - -p suites/target_test.function \ - -s suites \ - --helpers-file suites/helpers.function \ - -o ./TESTS/mbedtls/$* - -generate-target-tests: $(EMBEDDED_TESTS) - -define copy_header_to_target -TESTS/mbedtls/$(1)/$(2): include/test/$(2) - echo " Copy ./$$@" -ifndef WINDOWS - mkdir -p $$(@D) - cp $$< $$@ -else - mkdir $$(@D) - copy $$< $$@ -endif - -endef -$(foreach app, $(APPS), $(foreach file, $(notdir $(wildcard include/test/*.h)), \ - $(eval $(call copy_header_to_target,$(app),$(file))))) -$(addprefix embedded_,$(filter test_suite_psa_%, $(APPS))): embedded_%: $(patsubst TESTS/mbedtls/%, include/test/%, $(wildcard include/test/*. include/test/*/*.h)) - ifdef RECORD_PSA_STATUS_COVERAGE_LOG include/test/instrument_record_status.h: ../include/psa/crypto.h Makefile echo " Gen $@" diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py index 7382fb6ec..f5750aacf 100755 --- a/tests/scripts/generate_test_code.py +++ b/tests/scripts/generate_test_code.py @@ -106,10 +106,6 @@ Platform file: Platform file contains platform specific setup code and test case dispatch code. For example, host_test.function reads test data file from host's file system and dispatches tests. -In case of on-target target_test.function tests are not dispatched -on target. Target code is kept minimum and only test functions are -dispatched. Test case dispatch is done on the host using tools like -Greentea. Template file: --------- diff --git a/tests/scripts/mbedtls_test.py b/tests/scripts/mbedtls_test.py deleted file mode 100755 index 64f12bbb3..000000000 --- a/tests/scripts/mbedtls_test.py +++ /dev/null @@ -1,382 +0,0 @@ -#!/usr/bin/env python3 - -# Greentea host test script for Mbed TLS on-target test suite testing. -# -# Copyright The Mbed TLS Contributors -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -Mbed TLS on-target test suite tests are implemented as Greentea -tests. Greentea tests are implemented in two parts: target test and -host test. Target test is a C application that is built for the -target platform and executes on the target. Host test is a Python -class derived from mbed_host_tests.BaseHostTest. Target communicates -with the host over serial for the test data and sends back the result. - -Python tool mbedgt (Greentea) is responsible for flashing the test -binary on to the target and dynamically loading this host test module. - -Greentea documentation can be found here: -https://github.com/ARMmbed/greentea -""" - - -import re -import os -import binascii - -from mbed_host_tests import BaseHostTest, event_callback # type: ignore # pylint: disable=import-error - - -class TestDataParserError(Exception): - """Indicates error in test data, read from .data file.""" - pass - - -class TestDataParser: - """ - Parses test name, dependencies, test function name and test parameters - from the data file. - """ - - def __init__(self): - """ - Constructor - """ - self.tests = [] - - def parse(self, data_file): - """ - Data file parser. - - :param data_file: Data file path - """ - with open(data_file, 'r') as data_f: - self.__parse(data_f) - - @staticmethod - def __escaped_split(inp_str, split_char): - """ - Splits inp_str on split_char except when escaped. - - :param inp_str: String to split - :param split_char: Split character - :return: List of splits - """ - split_colon_fn = lambda x: re.sub(r'\\' + split_char, split_char, x) - if len(split_char) > 1: - raise ValueError('Expected split character. Found string!') - out = list(map(split_colon_fn, re.split(r'(?> x) & 0xff) for x in [24, 16, 8, 0]]) - return data_bytes - - def test_vector_to_bytes(self, function_id, dependencies, parameters): - """ - Converts test vector into a byte array that can be sent to the target. - - :param function_id: Test Function Identifier - :param dependencies: Dependency list - :param parameters: Test function input parameters - :return: Byte array and its length - """ - data_bytes = bytearray([len(dependencies)]) - if dependencies: - data_bytes += bytearray(dependencies) - data_bytes += bytearray([function_id, len(parameters)]) - for typ, param in parameters: - if typ in ('int', 'exp'): - i = int(param, 0) - data_bytes += b'I' if typ == 'int' else b'E' - self.align_32bit(data_bytes) - data_bytes += self.int32_to_big_endian_bytes(i) - elif typ == 'char*': - param = param.strip('"') - i = len(param) + 1 # + 1 for null termination - data_bytes += b'S' - self.align_32bit(data_bytes) - data_bytes += self.int32_to_big_endian_bytes(i) - data_bytes += bytearray(param, encoding='ascii') - data_bytes += b'\0' # Null terminate - elif typ == 'hex': - binary_data = self.hex_str_bytes(param) - data_bytes += b'H' - self.align_32bit(data_bytes) - i = len(binary_data) - data_bytes += self.int32_to_big_endian_bytes(i) - data_bytes += binary_data - length = self.int32_to_big_endian_bytes(len(data_bytes)) - return data_bytes, length - - def run_next_test(self): - """ - Fetch next test information and execute the test. - - """ - self.test_index += 1 - self.dep_index = 0 - if self.test_index < len(self.tests): - name, function_id, dependencies, args = self.tests[self.test_index] - self.run_test(name, function_id, dependencies, args) - else: - self.notify_complete(self.suite_passed) - - def run_test(self, name, function_id, dependencies, args): - """ - Execute the test on target by sending next test information. - - :param name: Test name - :param function_id: function identifier - :param dependencies: Dependencies list - :param args: test parameters - :return: - """ - self.log("Running: %s" % name) - - param_bytes, length = self.test_vector_to_bytes(function_id, - dependencies, args) - self.send_kv( - ''.join('{:02x}'.format(x) for x in length), - ''.join('{:02x}'.format(x) for x in param_bytes) - ) - - @staticmethod - def get_result(value): - """ - Converts result from string type to integer - :param value: Result code in string - :return: Integer result code. Value is from the test status - constants defined under the MbedTlsTest class. - """ - try: - return int(value) - except ValueError: - ValueError("Result should return error number. " - "Instead received %s" % value) - - @event_callback('GO') - def on_go(self, _key, _value, _timestamp): - """ - Sent by the target to start first test. - - :param _key: Event key - :param _value: Value. ignored - :param _timestamp: Timestamp ignored. - :return: - """ - self.run_next_test() - - @event_callback("R") - def on_result(self, _key, value, _timestamp): - """ - Handle result. Prints test start, finish required by Greentea - to detect test execution. - - :param _key: Event key - :param value: Value. ignored - :param _timestamp: Timestamp ignored. - :return: - """ - int_val = self.get_result(value) - name, _, _, _ = self.tests[self.test_index] - self.log('{{__testcase_start;%s}}' % name) - self.log('{{__testcase_finish;%s;%d;%d}}' % (name, int_val == 0, - int_val != 0)) - if int_val != 0: - self.suite_passed = False - self.run_next_test() - - @event_callback("F") - def on_failure(self, _key, value, _timestamp): - """ - Handles test execution failure. That means dependency not supported or - Test function not supported. Hence marking test as skipped. - - :param _key: Event key - :param value: Value. ignored - :param _timestamp: Timestamp ignored. - :return: - """ - int_val = self.get_result(value) - if int_val in self.error_str: - err = self.error_str[int_val] - else: - err = 'Unknown error' - # For skip status, do not write {{__testcase_finish;...}} - self.log("Error: %s" % err) - self.run_next_test() diff --git a/tests/suites/target_test.function b/tests/suites/target_test.function deleted file mode 100644 index 637a79d5e..000000000 --- a/tests/suites/target_test.function +++ /dev/null @@ -1,449 +0,0 @@ -#line 2 "suites/target_test.function" - -#include "greentea-client/test_env.h" - -/** - * \brief Increments pointer and asserts that it does not overflow. - * - * \param p Pointer to byte array - * \param start Pointer to start of byte array - * \param len Length of byte array - * \param step Increment size - * - */ -#define INCR_ASSERT(p, start, len, step) do \ -{ \ - TEST_HELPER_ASSERT( ( p ) >= ( start ) ); \ - TEST_HELPER_ASSERT( sizeof( *( p ) ) == sizeof( *( start ) ) ); \ - /* <= is checked to support use inside a loop where \ - pointer is incremented after reading data. */ \ - TEST_HELPER_ASSERT( (uint32_t)( ( ( p ) - ( start ) ) + ( step ) ) <= ( len ) );\ - ( p ) += ( step ); \ -} \ -while( 0 ) - - -/** - * \brief 4 byte align unsigned char pointer - * - * \param p Pointer to byte array - * \param start Pointer to start of byte array - * \param len Length of byte array - * - */ -#define ALIGN_32BIT(p, start, len) do \ -{ \ - uint32_t align = ( - (uintptr_t)( p ) ) % 4; \ - INCR_ASSERT( ( p ), ( start ), ( len ), align );\ -} \ -while( 0 ) - - -/** - * \brief Verify dependencies. Dependency identifiers are - * encoded in the buffer as 8 bit unsigned integers. - * - * \param count Number of dependencies. - * \param dep_p Pointer to buffer. - * - * \return DEPENDENCY_SUPPORTED if success else DEPENDENCY_NOT_SUPPORTED. - */ -int verify_dependencies( uint8_t count, uint8_t * dep_p ) -{ - uint8_t i; - for ( i = 0; i < count; i++ ) - { - if ( dep_check( (int)(dep_p[i]) ) != DEPENDENCY_SUPPORTED ) - return( DEPENDENCY_NOT_SUPPORTED ); - } - return( DEPENDENCY_SUPPORTED ); -} - -/** - * \brief Receives hex string on serial interface, and converts to a byte. - * - * \param none - * - * \return unsigned int8 - */ -uint8_t receive_byte() -{ - uint8_t byte; - uint8_t c[3]; - size_t len; - - c[0] = greentea_getc(); - c[1] = greentea_getc(); - c[2] = '\0'; - - TEST_HELPER_ASSERT( mbedtls_test_unhexify( &byte, sizeof( byte ), - c, &len ) == 0 ); - TEST_HELPER_ASSERT( len != 2 ); - - return( byte ); -} - -/** - * \brief Receives unsigned integer on serial interface. - * Integers are encoded in network order, and sent as hex ascii string. - * - * \param none - * - * \return unsigned int - */ -uint32_t receive_uint32() -{ - uint32_t value; - size_t len; - const uint8_t c_be[8] = { greentea_getc(), - greentea_getc(), - greentea_getc(), - greentea_getc(), - greentea_getc(), - greentea_getc(), - greentea_getc(), - greentea_getc() - }; - const uint8_t c[9] = { c_be[6], c_be[7], c_be[4], c_be[5], c_be[2], - c_be[3], c_be[0], c_be[1], '\0' }; - - TEST_HELPER_ASSERT( mbedtls_test_unhexify( (uint8_t*)&value, sizeof( value ), - c, &len ) == 0 ); - TEST_HELPER_ASSERT( len != 8 ); - - return( value ); -} - -/** - * \brief Parses out an unsigned 32 int value from the byte array. - * Integers are encoded in network order. - * - * \param p Pointer to byte array - * - * \return unsigned int - */ -uint32_t parse_uint32( uint8_t * p ) -{ - uint32_t value; - value = *p++ << 24; - value |= *p++ << 16; - value |= *p++ << 8; - value |= *p; - return( value ); -} - - -/** - * \brief Receives test data on serial as greentea key,value pair: - * {{;}} - * - * \param data_len Out pointer to hold received data length. - * - * \return Byte array. - */ -uint8_t * receive_data( uint32_t * data_len ) -{ - uint32_t i = 0, errors = 0; - char c; - uint8_t * data = NULL; - - /* Read opening braces */ - i = 0; - while ( i < 2 ) - { - c = greentea_getc(); - /* Ignore any prevous CR LF characters */ - if ( c == '\n' || c == '\r' ) - continue; - i++; - if ( c != '{' ) - return( NULL ); - } - - /* Read data length */ - *data_len = receive_uint32(); - data = (uint8_t *)malloc( *data_len ); - TEST_HELPER_ASSERT( data != NULL ); - - greentea_getc(); // read ';' received after key i.e. *data_len - - for( i = 0; i < *data_len; i++ ) - data[i] = receive_byte(); - - /* Read closing braces */ - for( i = 0; i < 2; i++ ) - { - c = greentea_getc(); - if ( c != '}' ) - { - errors++; - break; - } - } - - if ( errors ) - { - free( data ); - data = NULL; - *data_len = 0; - } - - return( data ); -} - -/** - * \brief Parse the received byte array and count the number of arguments - * to the test function passed as type hex. - * - * \param count Parameter count - * \param data Received Byte array - * \param data_len Byte array length - * - * \return count of hex params - */ -uint32_t find_hex_count( uint8_t count, uint8_t * data, uint32_t data_len ) -{ - uint32_t i = 0, sz = 0; - char c; - uint8_t * p = NULL; - uint32_t hex_count = 0; - - p = data; - - for( i = 0; i < count; i++ ) - { - c = (char)*p; - INCR_ASSERT( p, data, data_len, 1 ); - - /* Align p to 4 bytes for int, expression, string len or hex length */ - ALIGN_32BIT( p, data, data_len ); - - /* Network to host conversion */ - sz = (int32_t)parse_uint32( p ); - - INCR_ASSERT( p, data, data_len, sizeof( int32_t ) ); - - if ( c == 'H' || c == 'S' ) - { - INCR_ASSERT( p, data, data_len, sz ); - hex_count += ( c == 'H' )?1:0; - } - } - - return( hex_count ); -} - -/** - * \brief Parses received byte array for test parameters. - * - * \param count Parameter count - * \param data Received Byte array - * \param data_len Byte array length - * \param error Parsing error out variable. - * - * \return Array of parsed parameters allocated on heap. - * Note: Caller has the responsibility to delete - * the memory after use. - */ -void ** parse_parameters( uint8_t count, uint8_t * data, uint32_t data_len, - int * error ) -{ - uint32_t i = 0, hex_count = 0; - char c; - void ** params = NULL; - void ** cur = NULL; - uint8_t * p = NULL; - - hex_count = find_hex_count(count, data, data_len); - - params = (void **)malloc( sizeof( void *) * ( count + hex_count ) ); - TEST_HELPER_ASSERT( params != NULL ); - cur = params; - - p = data; - - /* Parameters */ - for( i = 0; i < count; i++ ) - { - c = (char)*p; - INCR_ASSERT( p, data, data_len, 1 ); - - /* Align p to 4 bytes for int, expression, string len or hex length */ - ALIGN_32BIT( p, data, data_len ); - - /* Network to host conversion */ - *( (int32_t *)p ) = (int32_t)parse_uint32( p ); - - switch( c ) - { - case 'E': - { - if ( get_expression( *( (int32_t *)p ), (int32_t *)p ) ) - { - *error = KEY_VALUE_MAPPING_NOT_FOUND; - goto exit; - } - } /* Intentional fall through */ - case 'I': - { - *cur++ = (void *)p; - INCR_ASSERT( p, data, data_len, sizeof( int32_t ) ); - } - break; - case 'H': /* Intentional fall through */ - case 'S': - { - uint32_t * sz = (uint32_t *)p; - INCR_ASSERT( p, data, data_len, sizeof( int32_t ) ); - *cur++ = (void *)p; - if ( c == 'H' ) - *cur++ = (void *)sz; - INCR_ASSERT( p, data, data_len, ( *sz ) ); - } - break; - default: - { - *error = DISPATCH_INVALID_TEST_DATA; - goto exit; - } - break; - } - } - -exit: - if ( *error ) - { - free( params ); - params = NULL; - } - - return( params ); -} - -/** - * \brief Sends greentea key and int value pair to host. - * - * \param key key string - * \param value integer value - * - * \return void - */ -void send_key_integer( char * key, int value ) -{ - char str[50]; - snprintf( str, sizeof( str ), "%d", value ); - greentea_send_kv( key, str ); -} - -/** - * \brief Sends test setup failure to the host. - * - * \param failure Test set failure - * - * \return void - */ -void send_failure( int failure ) -{ - send_key_integer( "F", failure ); -} - -/** - * \brief Sends test status to the host. - * - * \param status Test status (PASS=0/FAIL=!0) - * - * \return void - */ -void send_status( int status ) -{ - send_key_integer( "R", status ); -} - - -/** - * \brief Embedded implementation of execute_tests(). - * Ignores command line and received test data - * on serial. - * - * \param argc not used - * \param argv not used - * - * \return Program exit status. - */ -int execute_tests( int args, const char ** argv ) -{ - int ret = 0; - uint32_t data_len = 0; - uint8_t count = 0, function_id; - void ** params = NULL; - uint8_t * data = NULL, * p = NULL; - - GREENTEA_SETUP( 800, "mbedtls_test" ); - greentea_send_kv( "GO", " " ); - - while ( 1 ) - { - ret = 0; - mbedtls_test_info_reset( ); - data_len = 0; - - data = receive_data( &data_len ); - if ( data == NULL ) - continue; - p = data; - - do - { - /* Read dependency count */ - count = *p; - TEST_HELPER_ASSERT( count < data_len ); - INCR_ASSERT( p, data, data_len, sizeof( uint8_t ) ); - ret = verify_dependencies( count, p ); - if ( ret != DEPENDENCY_SUPPORTED ) - break; - - if ( count ) - INCR_ASSERT( p, data, data_len, count ); - - /* Read function id */ - function_id = *p; - INCR_ASSERT( p, data, data_len, sizeof( uint8_t ) ); - if ( ( ret = check_test( function_id ) ) != DISPATCH_TEST_SUCCESS ) - break; - - /* Read number of parameters */ - count = *p; - INCR_ASSERT( p, data, data_len, sizeof( uint8_t ) ); - - /* Parse parameters if present */ - if ( count ) - { - params = parse_parameters( count, p, data_len - ( p - data ), &ret ); - if ( ret ) - break; - } - - ret = dispatch_test( function_id, params ); - } - while ( 0 ); - - if ( data ) - { - free( data ); - data = NULL; - } - - if ( params ) - { - free( params ); - params = NULL; - } - - if ( ret ) - send_failure( ret ); - else - send_status( mbedtls_test_info.result ); - } - return( 0 ); -} -