From e78ee996248a8d9c96fc420f1fc3e2d4510cf581 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Wed, 22 Sep 2021 15:42:14 +0800 Subject: [PATCH 01/11] add enum value to string helpers Only add helpers for enum in `ssl.h`. Signed-off-by: Jerry Yu --- library/.gitignore | 2 + library/CMakeLists.txt | 1 + library/Makefile | 15 +- library/ssl_tls13_client.c | 1 + library/ssl_tls13_server.c | 1 + scripts/generate_ssl_debug_helpers.py | 310 +++++++++++++++++++++++++ scripts/make_generated_files.bat | 1 + tests/scripts/check-generated-files.sh | 1 + 8 files changed, 331 insertions(+), 1 deletion(-) create mode 100755 scripts/generate_ssl_debug_helpers.py diff --git a/library/.gitignore b/library/.gitignore index 6fde1f5e7..f6619d273 100644 --- a/library/.gitignore +++ b/library/.gitignore @@ -6,3 +6,5 @@ libmbed* # Automatically generated files /error.c /version_features.c +/ssl_debug_helpers_generated.c +/ssl_debug_helpers_generated.h diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index caac2d521..d04f9fd1b 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -77,6 +77,7 @@ set(src_crypto sha1.c sha256.c sha512.c + ssl_debug_helpers_generated.c threading.c timing.c version.c diff --git a/library/Makefile b/library/Makefile index bd116be82..2af90444e 100644 --- a/library/Makefile +++ b/library/Makefile @@ -24,6 +24,12 @@ endif PERL ?= perl +ifdef WINDOWS +PYTHON ?= python +else +PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; else echo python; fi) +endif + # if were running on Windows build for Windows ifdef WINDOWS WINDOWS_BUILD=1 @@ -136,6 +142,7 @@ OBJS_CRYPTO= \ sha1.o \ sha256.o \ sha512.o \ + ssl_debug_helpers_generated.o \ threading.o \ timing.o \ version.o \ @@ -281,7 +288,7 @@ libmbedcrypto.dll: $(OBJS_CRYPTO) $(CC) $(LOCAL_CFLAGS) $(CFLAGS) -o $@ -c $< .PHONY: generated_files -GENERATED_FILES = error.c version_features.c +GENERATED_FILES = error.c version_features.c ssl_debug_helpers_generated.c generated_files: $(GENERATED_FILES) error.c: ../scripts/generate_errors.pl @@ -291,6 +298,12 @@ error.c: echo " Gen $@" $(PERL) ../scripts/generate_errors.pl +ssl_debug_helpers_generated.c: ../scripts/generate_ssl_debug_helpers.py +ssl_debug_helpers_generated.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) +ssl_debug_helpers_generated.c: + echo " Gen $@" + $(PYTHON) ../scripts/generate_ssl_debug_helpers.py + version_features.c: ../scripts/generate_features.pl version_features.c: ../scripts/data_files/version_features.fmt ## The generated file only depends on the options that are present in mbedtls_config.h, diff --git a/library/ssl_tls13_client.c b/library/ssl_tls13_client.c index 1874d4fde..5968501e6 100644 --- a/library/ssl_tls13_client.c +++ b/library/ssl_tls13_client.c @@ -34,6 +34,7 @@ #include "ssl_misc.h" #include "ecdh_misc.h" #include "ssl_tls13_keys.h" +#include "ssl_debug_helpers_generated.h" /* Write extensions */ diff --git a/library/ssl_tls13_server.c b/library/ssl_tls13_server.c index 3018ecbcc..e9c74ca4c 100644 --- a/library/ssl_tls13_server.c +++ b/library/ssl_tls13_server.c @@ -26,6 +26,7 @@ #include "mbedtls/debug.h" #include "ssl_misc.h" +#include "ssl_debug_helpers_generated.h" int mbedtls_ssl_tls13_handshake_server_step( mbedtls_ssl_context *ssl ) { diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py new file mode 100755 index 000000000..9b626fe35 --- /dev/null +++ b/scripts/generate_ssl_debug_helpers.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +"""Generate library/ssl_debug_helps_generated.c + +The code generated by this module includes debug helper functions that can not be +implemented by fixed codes. + +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +import re +import os +import textwrap +from mbedtls_dev import build_tree + +def remove_c_comments(string): + """ + Remove C style comments from input string + """ + string_pattern = r"(?P\".*?\"|\'.*?\')" + comment_pattern = r"(?P/\*.*?\*/|//[^\r\n]*$)" + pattern = re.compile(string_pattern + r'|' + comment_pattern, + re.MULTILINE|re.DOTALL) + def replacer(match): + if match.lastgroup == 'comment': + return "" + return match.group() + return pattern.sub(replacer, string) + +class CondDirectiveNotMatch(Exception): + pass + +def preprocesse_c_source_code(source, *classes): + """ + Simple preprocessor for C source code. + + Only processs condition directives without expanding them. + Yield object accodring to the classes input. Most match firstly + + If there are directive pair does not match, raise CondDirectiveNotMatch. + + Assume source code does not include comments and compile pass. + + """ + + pattern = re.compile(r"^[ \t]*#[ \t]*" + + r"(?P(if[ \t]|ifndef[ \t]|ifdef[ \t]|else|endif))" + + r"[ \t]*(?P(.*\\\n)*.*$)", + re.MULTILINE) + stack = [] + + def _yield_objects(s, d, p, st, end): + """ + Output matched source piece + """ + nonlocal stack + start_line, end_line = '', '' + if stack: + start_line = '#{} {}'.format(d, p) + if d == 'if': + end_line = '#endif /* {} */'.format(p) + elif d == 'ifdef': + end_line = '#endif /* defined({}) */'.format(p) + else: + end_line = '#endif /* !defined({}) */'.format(p) + has_instance = False + for cls in classes: + for instance in cls.extract(s, st, end): + if has_instance is False: + has_instance = True + yield pair_start, start_line + yield instance.span()[0], instance + if has_instance: + yield start, end_line + + for match in pattern.finditer(source): + + directive = match.groupdict()['directive'].strip() + param = match.groupdict()['param'] + start, end = match.span() + + if directive in ('if', 'ifndef', 'ifdef'): + stack.append((directive, param, start, end)) + continue + + if not stack: + raise CondDirectiveNotMatch() + + pair_directive, pair_param, pair_start, pair_end = stack.pop() + yield from _yield_objects(source, + pair_directive, + pair_param, + pair_end, + start) + + if directive == 'endif': + continue + + if pair_directive == 'if': + directive = 'if' + param = "!( {} )".format(pair_param) + elif pair_directive == 'ifdef': + directive = 'ifndef' + param = pair_param + else: + directive = 'ifdef' + param = pair_param + + stack.append((directive, param, start, end)) + assert not stack, len(stack) + + + +class EnumDefinition: + """ + Generate helper functions around enumeration. + + Currently, it generate tranlation function from enum value to string. + Enum definition looks like: + [typedef] enum [prefix name] { [body] } [suffix name]; + + Known limitation: + - the '}' and ';' SHOULD NOT exist in different macro blocks. Like + ``` + enum test { + .... + #if defined(A) + .... + }; + #else + .... + }; + #endif + ``` + """ + + @classmethod + def extract(cls, source_code, start=0, end=-1): + enum_pattern = re.compile(r'enum\s*(?P\w*)\s*' + + r'{\s*(?P[^}]*)}' + + r'\s*(?P\w*)\s*;', + re.MULTILINE|re.DOTALL) + + for match in enum_pattern.finditer(source_code, start, end): + yield EnumDefinition(source_code, + span=match.span(), + group=match.groupdict()) + + def __init__(self, source_code, span=None, group=None): + assert isinstance(group, dict) + prefix_name = group.get('prefix_name', None) + suffix_name = group.get('suffix_name', None) + body = group.get('body', None) + assert prefix_name or suffix_name + assert body + assert span + # If suffix_name exists, it is a typedef + self._prototype = suffix_name if suffix_name else 'enum ' + prefix_name + self._name = suffix_name if suffix_name else prefix_name + self._body = body + self._source = source_code + self._span = span + + def __repr__(self): + return 'Enum({},{})'.format(self._name, self._span) + + def __str__(self): + return repr(self) + + def span(self): + return self._span + + def generate_tranlation_function(self): + """ + Generate function for translating value to string + """ + translation_table = [] + + for line in self._body.splitlines(): + + if line.strip().startswith('#'): + # Preprocess directive, keep it in table + translation_table.append(line.strip()) + continue + + if not line.strip(): + continue + + for field in line.strip().split(','): + if not field.strip(): + continue + member = field.strip().split()[0] + translation_table.append( + '{space}[{member}] = "{member}",'.format(member=member, + space=' '*8) + ) + + body = textwrap.dedent('''\ + const char *{name}_str( {prototype} in ) + {{ + const char * in_to_str[]= + {{ + {translation_table} + }}; + + if( in > ( sizeof( in_to_str )/sizeof( in_to_str[0]) - 1 ) || + in_to_str[ in ] == NULL ) + {{ + return "UNKOWN_VAULE"; + }} + return in_to_str[ in ]; + }} + ''') + body = body.format(translation_table='\n'.join(translation_table), + name=self._name, + prototype=self._prototype) + prototype = 'const char *{name}_str( {prototype} in );\n' + prototype = prototype.format(name=self._name, + prototype=self._prototype) + return body, prototype + +OUTPUT_C_TEMPLATE = '''\ +/* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */ + +#include "common.h" + +#if defined(MBEDTLS_DEBUG_C) + +#include "ssl_debug_helpers_generated.h" + +{functions} + +#endif /* MBEDTLS_DEBUG_C */ +/* End of automatically generated file. */ + +''' + +OUTPUT_H_TEMPLATE = '''\ +/* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */ +#ifndef MBEDTLS_SSL_DEBUG_HELPERS_H +#define MBEDTLS_SSL_DEBUG_HELPERS_H + +#include "common.h" + +#if defined(MBEDTLS_DEBUG_C) + +#include "mbedtls/ssl.h" +#include "ssl_misc.h" + +{functions} + +#endif /* MBEDTLS_DEBUG_C */ + +#endif /* SSL_DEBUG_HELPERS_H */ + +/* End of automatically generated file. */ + +''' + + +def generate_ssl_debug_helpers(target_dir): + """ + Generate functions of debug helps + """ + with open('include/mbedtls/ssl.h') as f: + source_code = remove_c_comments(f.read()) + + definitions = dict() + prototypes = dict() + for start, instance in preprocesse_c_source_code(source_code, EnumDefinition): + if start in definitions: + continue + if isinstance(instance, EnumDefinition): + definition, prototype = instance.generate_tranlation_function() + else: + definition = instance + prototype = instance + definitions[start] = definition + prototypes[start] = prototype + + functions = [str(v) for _, v in sorted(definitions.items())] + with open(os.path.join(target_dir, 'ssl_debug_helpers_generated.c'), 'w') as f: + f.write(OUTPUT_C_TEMPLATE.format(functions='\n'.join(functions))) + + functions = [str(v) for _, v in sorted(prototypes.items())] + with open(os.path.join(target_dir, 'ssl_debug_helpers_generated.h'), 'w') as f: + f.write(OUTPUT_H_TEMPLATE.format(functions='\n'.join(functions))) + + + + + +if __name__ == '__main__': + build_tree.chdir_to_root() + OUTPUT_FILE_DIR = sys.argv[1] if len(sys.argv) == 2 else "library" + generate_ssl_debug_helpers(OUTPUT_FILE_DIR) diff --git a/scripts/make_generated_files.bat b/scripts/make_generated_files.bat index e4465d8f0..d3a8b3645 100644 --- a/scripts/make_generated_files.bat +++ b/scripts/make_generated_files.bat @@ -4,6 +4,7 @@ perl scripts\generate_errors.pl || exit /b 1 perl scripts\generate_query_config.pl || exit /b 1 perl scripts\generate_features.pl || exit /b 1 +python scripts\generate_ssl_debug_helpers.py || exit /b 1 perl scripts\generate_visualc_files.pl || exit /b 1 python scripts\generate_psa_constants.py || exit /b 1 python tests\scripts\generate_psa_tests.py || exit /b 1 diff --git a/tests/scripts/check-generated-files.sh b/tests/scripts/check-generated-files.sh index 0399484d5..994fd243b 100755 --- a/tests/scripts/check-generated-files.sh +++ b/tests/scripts/check-generated-files.sh @@ -118,6 +118,7 @@ check() check scripts/generate_errors.pl library/error.c check scripts/generate_query_config.pl programs/test/query_config.c check scripts/generate_features.pl library/version_features.c +check scripts/generate_ssl_debug_helpers.py library/ssl_debug_helpers_generated.c # generate_visualc_files enumerates source files (library/*.c). It doesn't # care about their content, but the files must exist. So it must run after # the step that creates or updates these files. From e3b3412bc407da12eeba95e7f2960f48b7364f46 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Tue, 28 Sep 2021 17:53:35 +0800 Subject: [PATCH 02/11] Add tests for enum helper Signed-off-by: Jerry Yu --- library/ssl_tls13_client.c | 4 +++- library/ssl_tls13_server.c | 4 +++- tests/ssl-opt.sh | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/library/ssl_tls13_client.c b/library/ssl_tls13_client.c index 5968501e6..d401aea6c 100644 --- a/library/ssl_tls13_client.c +++ b/library/ssl_tls13_client.c @@ -1663,7 +1663,9 @@ int mbedtls_ssl_tls13_handshake_client_step( mbedtls_ssl_context *ssl ) { int ret = 0; - MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 client state: %d", ssl->state ) ); + MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 client state: %s(%d)", + mbedtls_ssl_states_str( ssl->state ), + ssl->state ) ); switch( ssl->state ) { diff --git a/library/ssl_tls13_server.c b/library/ssl_tls13_server.c index e9c74ca4c..67c072534 100644 --- a/library/ssl_tls13_server.c +++ b/library/ssl_tls13_server.c @@ -31,7 +31,9 @@ int mbedtls_ssl_tls13_handshake_server_step( mbedtls_ssl_context *ssl ) { ((void) ssl); - MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 server state: %d", ssl->state ) ); + MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 server state: %s(%d)", + mbedtls_ssl_states_str( ssl->state ), + ssl->state ) ); return( MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE ); } diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index d5e9e0199..eadb2111f 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -8809,8 +8809,8 @@ run_test "TLS 1.3: handshake dispatch test: tls13 only" \ "$P_SRV debug_level=2 min_version=tls13 max_version=tls13" \ "$P_CLI debug_level=2 min_version=tls13 max_version=tls13" \ 1 \ - -s "tls13 server state: 0" \ - -c "tls13 client state: 0" + -s "tls13 server state: MBEDTLS_SSL_HELLO_REQUEST" \ + -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST" requires_openssl_tls1_3 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL From e988f0f7a74fab6fdd969452947b6f419940eddf Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 11 Nov 2021 13:22:20 +0800 Subject: [PATCH 03/11] fix wrong typo Signed-off-by: Jerry Yu --- scripts/generate_ssl_debug_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py index 9b626fe35..3065d08f2 100755 --- a/scripts/generate_ssl_debug_helpers.py +++ b/scripts/generate_ssl_debug_helpers.py @@ -44,7 +44,7 @@ def remove_c_comments(string): class CondDirectiveNotMatch(Exception): pass -def preprocesse_c_source_code(source, *classes): +def preprocess_c_source_code(source, *classes): """ Simple preprocessor for C source code. @@ -281,7 +281,7 @@ def generate_ssl_debug_helpers(target_dir): definitions = dict() prototypes = dict() - for start, instance in preprocesse_c_source_code(source_code, EnumDefinition): + for start, instance in preprocess_c_source_code(source_code, EnumDefinition): if start in definitions: continue if isinstance(instance, EnumDefinition): From cdcc55f46fab11cde9c8e7001a12aba9f7eb4cfc Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 11 Nov 2021 13:26:33 +0800 Subject: [PATCH 04/11] update test check strings Signed-off-by: Jerry Yu --- tests/ssl-opt.sh | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index eadb2111f..c54a6adcb 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -8817,19 +8817,19 @@ requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL requires_config_disabled MBEDTLS_USE_PSA_CRYPTO run_test "TLS 1.3: minimal feature sets - openssl" \ "$O_NEXT_SRV -msg -tls1_3 -no_middlebox -num_tickets 0 -no_resume_ephemeral -no_cache" \ - "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \ + "$P_CLI debug_level=3 min_version=tls1_3 max_version=tls1_3" \ 0 \ - -c "tls13 client state: 0" \ - -c "tls13 client state: 2" \ - -c "tls13 client state: 19" \ - -c "tls13 client state: 5" \ - -c "tls13 client state: 3" \ - -c "tls13 client state: 9" \ - -c "tls13 client state: 13" \ - -c "tls13 client state: 11" \ - -c "tls13 client state: 14" \ - -c "tls13 client state: 15" \ - -c "<= ssl_tls13_process_server_hello" \ + -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_HELLO(2)" \ + -c "tls13 client state: MBEDTLS_SSL_ENCRYPTED_EXTENSIONS(19)" \ + -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_REQUEST(5)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_CERTIFICATE(3)" \ + -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_VERIFY(9)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_FINISHED(13)" \ + -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)" \ + -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)" \ + -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)" \ + -c "<= ssl_tls1_3_process_server_hello" \ -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \ -c "ECDH curve: x25519" \ -c "=> ssl_tls13_process_server_hello" \ @@ -8848,20 +8848,20 @@ requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL requires_config_disabled MBEDTLS_USE_PSA_CRYPTO run_test "TLS 1.3: minimal feature sets - gnutls" \ "$G_NEXT_SRV --debug=4 --priority=NORMAL:-VERS-ALL:+VERS-TLS1.3:+CIPHER-ALL:%NO_TICKETS:%DISABLE_TLS13_COMPAT_MODE --disable-client-cert" \ - "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \ + "$P_CLI debug_level=3 min_version=tls1_3 max_version=tls1_3" \ 0 \ - -s "SERVER HELLO was queued" \ - -c "tls13 client state: 0" \ - -c "tls13 client state: 2" \ - -c "tls13 client state: 19" \ - -c "tls13 client state: 5" \ - -c "tls13 client state: 3" \ - -c "tls13 client state: 9" \ - -c "tls13 client state: 13" \ - -c "tls13 client state: 11" \ - -c "tls13 client state: 14" \ - -c "tls13 client state: 15" \ - -c "<= ssl_tls13_process_server_hello" \ + -s "SERVER HELLO was queued" \ + -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_HELLO(2)" \ + -c "tls13 client state: MBEDTLS_SSL_ENCRYPTED_EXTENSIONS(19)" \ + -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_REQUEST(5)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_CERTIFICATE(3)" \ + -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_VERIFY(9)" \ + -c "tls13 client state: MBEDTLS_SSL_SERVER_FINISHED(13)" \ + -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)" \ + -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)" \ + -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)" \ + -c "<= ssl_tls1_3_process_server_hello" \ -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \ -c "ECDH curve: x25519" \ -c "=> ssl_tls13_process_server_hello" \ From 6389b254c9141437c57e8544410fe5de01d0f88e Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 10:28:40 +0800 Subject: [PATCH 05/11] fix typos error Signed-off-by: Jerry Yu --- scripts/generate_ssl_debug_helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py index 3065d08f2..711bfd398 100755 --- a/scripts/generate_ssl_debug_helpers.py +++ b/scripts/generate_ssl_debug_helpers.py @@ -48,10 +48,10 @@ def preprocess_c_source_code(source, *classes): """ Simple preprocessor for C source code. - Only processs condition directives without expanding them. - Yield object accodring to the classes input. Most match firstly + Only processses condition directives without expanding them. + Yield object according to the classes input. Most match firstly - If there are directive pair does not match, raise CondDirectiveNotMatch. + If the directive pair does not match , raise CondDirectiveNotMatch. Assume source code does not include comments and compile pass. @@ -129,7 +129,7 @@ class EnumDefinition: """ Generate helper functions around enumeration. - Currently, it generate tranlation function from enum value to string. + Currently, it generate translation function from enum value to string. Enum definition looks like: [typedef] enum [prefix name] { [body] } [suffix name]; From eb96fb508e384d878fe694d0b13863da8f2a7760 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 11:03:59 +0800 Subject: [PATCH 06/11] Add cmake generator Signed-off-by: Jerry Yu --- library/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index d04f9fd1b..0fb46d65d 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -144,9 +144,21 @@ if(GEN_FILES) ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls/mbedtls_config.h ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files/version_features.fmt ) + + add_custom_command( + OUTPUT + ${CMAKE_CURRENT_BINARY_DIR}/ssl_debug_helpers_generated.c + COMMAND + ${MBEDTLS_PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py + DEPENDS + ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py + ${error_headers} + ) else() link_to_source(error.c) link_to_source(version_features.c) + link_to_source(ssl_debug_helpers_generated.c) endif() if(CMAKE_COMPILER_IS_GNUCC) From e6369b0061c4d4f30bf3a16faa98cbfc04363615 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 13:51:26 +0800 Subject: [PATCH 07/11] fix test_cmake_as_package fail Signed-off-by: Jerry Yu --- library/CMakeLists.txt | 2 ++ scripts/generate_ssl_debug_helpers.py | 52 +++++++++++++++++++-------- scripts/mbedtls_dev/build_tree.py | 21 +++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 0fb46d65d..add078413 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -151,6 +151,8 @@ if(GEN_FILES) COMMAND ${MBEDTLS_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py + --mbedtls-root ${CMAKE_CURRENT_SOURCE_DIR}/.. + ${CMAKE_CURRENT_BINARY_DIR} DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py ${error_headers} diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py index 711bfd398..ad1f94997 100755 --- a/scripts/generate_ssl_debug_helpers.py +++ b/scripts/generate_ssl_debug_helpers.py @@ -25,8 +25,10 @@ import sys import re import os import textwrap +import argparse from mbedtls_dev import build_tree + def remove_c_comments(string): """ Remove C style comments from input string @@ -34,16 +36,19 @@ def remove_c_comments(string): string_pattern = r"(?P\".*?\"|\'.*?\')" comment_pattern = r"(?P/\*.*?\*/|//[^\r\n]*$)" pattern = re.compile(string_pattern + r'|' + comment_pattern, - re.MULTILINE|re.DOTALL) + re.MULTILINE | re.DOTALL) + def replacer(match): if match.lastgroup == 'comment': return "" return match.group() return pattern.sub(replacer, string) + class CondDirectiveNotMatch(Exception): pass + def preprocess_c_source_code(source, *classes): """ Simple preprocessor for C source code. @@ -124,7 +129,6 @@ def preprocess_c_source_code(source, *classes): assert not stack, len(stack) - class EnumDefinition: """ Generate helper functions around enumeration. @@ -153,7 +157,7 @@ class EnumDefinition: enum_pattern = re.compile(r'enum\s*(?P\w*)\s*' + r'{\s*(?P[^}]*)}' + r'\s*(?P\w*)\s*;', - re.MULTILINE|re.DOTALL) + re.MULTILINE | re.DOTALL) for match in enum_pattern.finditer(source_code, start, end): yield EnumDefinition(source_code, @@ -233,6 +237,7 @@ class EnumDefinition: prototype=self._prototype) return body, prototype + OUTPUT_C_TEMPLATE = '''\ /* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */ @@ -272,11 +277,11 @@ OUTPUT_H_TEMPLATE = '''\ ''' -def generate_ssl_debug_helpers(target_dir): +def generate_ssl_debug_helpers(output_directory, mbedtls_root): """ Generate functions of debug helps """ - with open('include/mbedtls/ssl.h') as f: + with open(os.path.join(mbedtls_root, 'include/mbedtls/ssl.h')) as f: source_code = remove_c_comments(f.read()) definitions = dict() @@ -292,19 +297,38 @@ def generate_ssl_debug_helpers(target_dir): definitions[start] = definition prototypes[start] = prototype - functions = [str(v) for _, v in sorted(definitions.items())] - with open(os.path.join(target_dir, 'ssl_debug_helpers_generated.c'), 'w') as f: - f.write(OUTPUT_C_TEMPLATE.format(functions='\n'.join(functions))) + function_definitions = [str(v) for _, v in sorted(definitions.items())] + function_prototypes = [str(v) for _, v in sorted(prototypes.items())] + if output_directory == sys.stdout: + sys.stdout.write(OUTPUT_H_TEMPLATE.format( + functions='\n'.join(function_prototypes))) + sys.stdout.write(OUTPUT_C_TEMPLATE.format( + functions='\n'.join(function_definitions))) + else: + with open(os.path.join(output_directory, 'ssl_debug_helpers_generated.c'), 'w') as f: + f.write(OUTPUT_C_TEMPLATE.format( + functions='\n'.join(function_definitions))) - functions = [str(v) for _, v in sorted(prototypes.items())] - with open(os.path.join(target_dir, 'ssl_debug_helpers_generated.h'), 'w') as f: - f.write(OUTPUT_H_TEMPLATE.format(functions='\n'.join(functions))) + with open(os.path.join(output_directory, 'ssl_debug_helpers_generated.h'), 'w') as f: + f.write(OUTPUT_H_TEMPLATE.format( + functions='\n'.join(function_prototypes))) +def main(): + """ + Command line entry + """ + parser = argparse.ArgumentParser() + parser.add_argument('--mbedtls-root', nargs='?', default=build_tree.guess_mbedtls_root(), + help='root directory of mbedtls source code') + parser.add_argument('output_directory', nargs='?', + default=sys.stdout, help='source/header files location') + args = parser.parse_args() + + generate_ssl_debug_helpers(args.output_directory, args.mbedtls_root) + return 0 if __name__ == '__main__': - build_tree.chdir_to_root() - OUTPUT_FILE_DIR = sys.argv[1] if len(sys.argv) == 2 else "library" - generate_ssl_debug_helpers(OUTPUT_FILE_DIR) + sys.exit(main()) diff --git a/scripts/mbedtls_dev/build_tree.py b/scripts/mbedtls_dev/build_tree.py index 772410473..aee68f140 100644 --- a/scripts/mbedtls_dev/build_tree.py +++ b/scripts/mbedtls_dev/build_tree.py @@ -17,12 +17,15 @@ # limitations under the License. import os +import inspect + def looks_like_mbedtls_root(path: str) -> bool: """Whether the given directory looks like the root of the Mbed TLS source tree.""" return all(os.path.isdir(os.path.join(path, subdir)) for subdir in ['include', 'library', 'programs', 'tests']) + def chdir_to_root() -> None: """Detect the root of the Mbed TLS source tree and change to it. @@ -36,3 +39,21 @@ def chdir_to_root() -> None: os.chdir(d) return raise Exception('Mbed TLS source tree not found') + + +def guess_mbedtls_root(): + """Guess mbedTLS source code directory. + + Return the first possible mbedTLS root directory + """ + dirs = set({}) + for i in inspect.stack(): + path = os.path.dirname(i.filename) + for d in ['.', os.path.pardir, os.path.join(*([os.path.pardir]*2))]: + d = os.path.abspath(os.path.join(path, d)) + if d in dirs: + continue + dirs.add(d) + if looks_like_mbedtls_root(d): + return d + raise Exception('Mbed TLS source tree not found') From d05e1cec4b2346e5c6acb1db01c85aea7bba6421 Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 14:09:22 +0800 Subject: [PATCH 08/11] fix build fail on `check_*` Signed-off-by: Jerry Yu --- library/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/Makefile b/library/Makefile index 2af90444e..9881eb7a6 100644 --- a/library/Makefile +++ b/library/Makefile @@ -302,7 +302,7 @@ ssl_debug_helpers_generated.c: ../scripts/generate_ssl_debug_helpers.py ssl_debug_helpers_generated.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) ssl_debug_helpers_generated.c: echo " Gen $@" - $(PYTHON) ../scripts/generate_ssl_debug_helpers.py + $(PYTHON) ../scripts/generate_ssl_debug_helpers.py --mbedtls-root .. . version_features.c: ../scripts/generate_features.pl version_features.c: ../scripts/data_files/version_features.fmt From 9817e3e6217f4380b3134e668bf8c23eb5e410de Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Thu, 2 Dec 2021 14:32:58 +0800 Subject: [PATCH 09/11] fix generate check fail Signed-off-by: Jerry Yu --- scripts/generate_ssl_debug_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py index ad1f94997..fb726901d 100755 --- a/scripts/generate_ssl_debug_helpers.py +++ b/scripts/generate_ssl_debug_helpers.py @@ -322,7 +322,7 @@ def main(): parser.add_argument('--mbedtls-root', nargs='?', default=build_tree.guess_mbedtls_root(), help='root directory of mbedtls source code') parser.add_argument('output_directory', nargs='?', - default=sys.stdout, help='source/header files location') + default='library', help='source/header files location') args = parser.parse_args() From 0cb2cf6cb4753c4e8d247117a5e9cc2fb0296cee Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Fri, 10 Dec 2021 14:21:27 +0800 Subject: [PATCH 10/11] fix build fail Signed-off-by: Jerry Yu --- scripts/generate_ssl_debug_helpers.py | 3 ++- scripts/mbedtls_dev/build_tree.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/generate_ssl_debug_helpers.py b/scripts/generate_ssl_debug_helpers.py index fb726901d..19fac656c 100755 --- a/scripts/generate_ssl_debug_helpers.py +++ b/scripts/generate_ssl_debug_helpers.py @@ -281,6 +281,7 @@ def generate_ssl_debug_helpers(output_directory, mbedtls_root): """ Generate functions of debug helps """ + mbedtls_root = os.path.abspath(mbedtls_root or build_tree.guess_mbedtls_root()) with open(os.path.join(mbedtls_root, 'include/mbedtls/ssl.h')) as f: source_code = remove_c_comments(f.read()) @@ -319,7 +320,7 @@ def main(): Command line entry """ parser = argparse.ArgumentParser() - parser.add_argument('--mbedtls-root', nargs='?', default=build_tree.guess_mbedtls_root(), + parser.add_argument('--mbedtls-root', nargs='?', default=None, help='root directory of mbedtls source code') parser.add_argument('output_directory', nargs='?', default='library', help='source/header files location') diff --git a/scripts/mbedtls_dev/build_tree.py b/scripts/mbedtls_dev/build_tree.py index aee68f140..3920d0ed6 100644 --- a/scripts/mbedtls_dev/build_tree.py +++ b/scripts/mbedtls_dev/build_tree.py @@ -47,9 +47,10 @@ def guess_mbedtls_root(): Return the first possible mbedTLS root directory """ dirs = set({}) - for i in inspect.stack(): - path = os.path.dirname(i.filename) - for d in ['.', os.path.pardir, os.path.join(*([os.path.pardir]*2))]: + for frame in inspect.stack(): + path = os.path.dirname(frame.filename) + for d in ['.', os.path.pardir] \ + + [os.path.join(*([os.path.pardir]*i)) for i in range(2, 10)]: d = os.path.abspath(os.path.join(path, d)) if d in dirs: continue From bc8b22ecc82f32d5cfe036996df1c1734377bcbb Mon Sep 17 00:00:00 2001 From: Jerry Yu Date: Fri, 10 Dec 2021 15:33:20 +0800 Subject: [PATCH 11/11] fix tls13 test fail Signed-off-by: Jerry Yu --- tests/ssl-opt.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index c54a6adcb..fb396e525 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -8817,7 +8817,7 @@ requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL requires_config_disabled MBEDTLS_USE_PSA_CRYPTO run_test "TLS 1.3: minimal feature sets - openssl" \ "$O_NEXT_SRV -msg -tls1_3 -no_middlebox -num_tickets 0 -no_resume_ephemeral -no_cache" \ - "$P_CLI debug_level=3 min_version=tls1_3 max_version=tls1_3" \ + "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \ 0 \ -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)" \ -c "tls13 client state: MBEDTLS_SSL_SERVER_HELLO(2)" \ @@ -8829,7 +8829,7 @@ run_test "TLS 1.3: minimal feature sets - openssl" \ -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)" \ -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)" \ -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)" \ - -c "<= ssl_tls1_3_process_server_hello" \ + -c "<= ssl_tls13_process_server_hello" \ -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \ -c "ECDH curve: x25519" \ -c "=> ssl_tls13_process_server_hello" \ @@ -8848,7 +8848,7 @@ requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL requires_config_disabled MBEDTLS_USE_PSA_CRYPTO run_test "TLS 1.3: minimal feature sets - gnutls" \ "$G_NEXT_SRV --debug=4 --priority=NORMAL:-VERS-ALL:+VERS-TLS1.3:+CIPHER-ALL:%NO_TICKETS:%DISABLE_TLS13_COMPAT_MODE --disable-client-cert" \ - "$P_CLI debug_level=3 min_version=tls1_3 max_version=tls1_3" \ + "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \ 0 \ -s "SERVER HELLO was queued" \ -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)" \ @@ -8861,7 +8861,7 @@ run_test "TLS 1.3: minimal feature sets - gnutls" \ -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)" \ -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)" \ -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)" \ - -c "<= ssl_tls1_3_process_server_hello" \ + -c "<= ssl_tls13_process_server_hello" \ -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \ -c "ECDH curve: x25519" \ -c "=> ssl_tls13_process_server_hello" \