Merge pull request #4981 from yuhaoth/pr/add-debug-helpers-generated

Add debug helpers generated

library/.gitignore

@@ -6,3 +6,5 @@ libmbed*
 # Automatically generated files
 /error.c
 /version_features.c
+/ssl_debug_helpers_generated.c
+/ssl_debug_helpers_generated.h

library/CMakeLists.txt

@@ -77,6 +77,7 @@ set(src_crypto
     sha1.c
     sha256.c
     sha512.c
+    ssl_debug_helpers_generated.c
     threading.c
     timing.c
     version.c
@@ -143,9 +144,23 @@ if(GEN_FILES)
             ${CMAKE_CURRENT_SOURCE_DIR}/../include/mbedtls/mbedtls_config.h
             ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/data_files/version_features.fmt
     )
+
+    add_custom_command(
+        OUTPUT
+            ${CMAKE_CURRENT_BINARY_DIR}/ssl_debug_helpers_generated.c
+        COMMAND
+            ${MBEDTLS_PYTHON_EXECUTABLE}
+                ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py
+                --mbedtls-root ${CMAKE_CURRENT_SOURCE_DIR}/..
+                ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDS
+            ${CMAKE_CURRENT_SOURCE_DIR}/../scripts/generate_ssl_debug_helpers.py
+            ${error_headers}
+    )
 else()
     link_to_source(error.c)
     link_to_source(version_features.c)
+    link_to_source(ssl_debug_helpers_generated.c)
 endif()
 
 if(CMAKE_COMPILER_IS_GNUCC)

library/Makefile

@@ -24,6 +24,12 @@ endif
 
 PERL ?= perl
 
+ifdef WINDOWS
+PYTHON ?= python
+else
+PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; else echo python; fi)
+endif
+
 # if were running on Windows build for Windows
 ifdef WINDOWS
 WINDOWS_BUILD=1
@@ -136,6 +142,7 @@ OBJS_CRYPTO= \
 	     sha1.o \
 	     sha256.o \
 	     sha512.o \
+	     ssl_debug_helpers_generated.o \
 	     threading.o \
 	     timing.o \
 	     version.o \
@@ -281,7 +288,7 @@ libmbedcrypto.dll: $(OBJS_CRYPTO)
 	$(CC) $(LOCAL_CFLAGS) $(CFLAGS) -o $@ -c $<
 
 .PHONY: generated_files
-GENERATED_FILES = error.c version_features.c
+GENERATED_FILES = error.c version_features.c ssl_debug_helpers_generated.c
 generated_files: $(GENERATED_FILES)
 
 error.c: ../scripts/generate_errors.pl
@@ -291,6 +298,12 @@ error.c:
 	echo "  Gen   $@"
 	$(PERL) ../scripts/generate_errors.pl
 
+ssl_debug_helpers_generated.c: ../scripts/generate_ssl_debug_helpers.py
+ssl_debug_helpers_generated.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h))
+ssl_debug_helpers_generated.c:
+	echo "  Gen   $@"
+	$(PYTHON) ../scripts/generate_ssl_debug_helpers.py --mbedtls-root .. .
+
 version_features.c: ../scripts/generate_features.pl
 version_features.c: ../scripts/data_files/version_features.fmt
 ## The generated file only depends on the options that are present in mbedtls_config.h,

library/ssl_tls13_client.c

@@ -34,6 +34,7 @@
 #include "ssl_misc.h"
 #include "ecdh_misc.h"
 #include "ssl_tls13_keys.h"
+#include "ssl_debug_helpers_generated.h"
 
 /* Write extensions */
 
@@ -1717,7 +1718,9 @@ int mbedtls_ssl_tls13_handshake_client_step( mbedtls_ssl_context *ssl )
 {
     int ret = 0;
 
-    MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 client state: %d", ssl->state ) );
+    MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 client state: %s(%d)",
+                                mbedtls_ssl_states_str( ssl->state ),
+                                ssl->state ) );
 
     switch( ssl->state )
     {

library/ssl_tls13_server.c

@@ -26,11 +26,14 @@
 #include "mbedtls/debug.h"
 
 #include "ssl_misc.h"
+#include "ssl_debug_helpers_generated.h"
 
 int mbedtls_ssl_tls13_handshake_server_step( mbedtls_ssl_context *ssl )
 {
     ((void) ssl);
-    MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 server state: %d", ssl->state ) );
+    MBEDTLS_SSL_DEBUG_MSG( 2, ( "tls13 server state: %s(%d)",
+                                mbedtls_ssl_states_str( ssl->state ),
+                                ssl->state ) );
 
     return( MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE );
 }

scripts/generate_ssl_debug_helpers.py

@@ -0,0 +1,335 @@
+#!/usr/bin/env python3
+
+"""Generate library/ssl_debug_helps_generated.c
+
+The code generated by this module includes debug helper functions that can not be
+implemented by fixed codes.
+
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+import re
+import os
+import textwrap
+import argparse
+from mbedtls_dev import build_tree
+
+
+def remove_c_comments(string):
+    """
+        Remove C style comments from input string
+    """
+    string_pattern = r"(?P<string>\".*?\"|\'.*?\')"
+    comment_pattern = r"(?P<comment>/\*.*?\*/|//[^\r\n]*$)"
+    pattern = re.compile(string_pattern + r'|' + comment_pattern,
+                         re.MULTILINE | re.DOTALL)
+
+    def replacer(match):
+        if match.lastgroup == 'comment':
+            return ""
+        return match.group()
+    return pattern.sub(replacer, string)
+
+
+class CondDirectiveNotMatch(Exception):
+    pass
+
+
+def preprocess_c_source_code(source, *classes):
+    """
+        Simple preprocessor for C source code.
+
+        Only processses condition directives without expanding them.
+        Yield object according to the classes input. Most match firstly
+
+        If the directive pair does not match , raise CondDirectiveNotMatch.
+
+        Assume source code does not include comments and compile pass.
+
+    """
+
+    pattern = re.compile(r"^[ \t]*#[ \t]*" +
+                         r"(?P<directive>(if[ \t]|ifndef[ \t]|ifdef[ \t]|else|endif))" +
+                         r"[ \t]*(?P<param>(.*\\\n)*.*$)",
+                         re.MULTILINE)
+    stack = []
+
+    def _yield_objects(s, d, p, st, end):
+        """
+            Output matched source piece
+        """
+        nonlocal stack
+        start_line, end_line = '', ''
+        if stack:
+            start_line = '#{} {}'.format(d, p)
+            if d == 'if':
+                end_line = '#endif /* {} */'.format(p)
+            elif d == 'ifdef':
+                end_line = '#endif /* defined({}) */'.format(p)
+            else:
+                end_line = '#endif /* !defined({}) */'.format(p)
+        has_instance = False
+        for cls in classes:
+            for instance in cls.extract(s, st, end):
+                if has_instance is False:
+                    has_instance = True
+                    yield pair_start, start_line
+                yield instance.span()[0], instance
+        if has_instance:
+            yield start, end_line
+
+    for match in pattern.finditer(source):
+
+        directive = match.groupdict()['directive'].strip()
+        param = match.groupdict()['param']
+        start, end = match.span()
+
+        if directive in ('if', 'ifndef', 'ifdef'):
+            stack.append((directive, param, start, end))
+            continue
+
+        if not stack:
+            raise CondDirectiveNotMatch()
+
+        pair_directive, pair_param, pair_start, pair_end = stack.pop()
+        yield from _yield_objects(source,
+                                  pair_directive,
+                                  pair_param,
+                                  pair_end,
+                                  start)
+
+        if directive == 'endif':
+            continue
+
+        if pair_directive == 'if':
+            directive = 'if'
+            param = "!( {} )".format(pair_param)
+        elif pair_directive == 'ifdef':
+            directive = 'ifndef'
+            param = pair_param
+        else:
+            directive = 'ifdef'
+            param = pair_param
+
+        stack.append((directive, param, start, end))
+    assert not stack, len(stack)
+
+
+class EnumDefinition:
+    """
+        Generate helper functions around enumeration.
+
+        Currently, it generate translation function from enum value to string.
+        Enum definition looks like:
+        [typedef] enum [prefix name] { [body] } [suffix name];
+
+        Known limitation:
+        - the '}' and ';' SHOULD NOT exist in different macro blocks. Like
+        ```
+        enum test {
+            ....
+        #if defined(A)
+            ....
+        };
+        #else
+            ....
+        };
+        #endif
+        ```
+    """
+
+    @classmethod
+    def extract(cls, source_code, start=0, end=-1):
+        enum_pattern = re.compile(r'enum\s*(?P<prefix_name>\w*)\s*' +
+                                  r'{\s*(?P<body>[^}]*)}' +
+                                  r'\s*(?P<suffix_name>\w*)\s*;',
+                                  re.MULTILINE | re.DOTALL)
+
+        for match in enum_pattern.finditer(source_code, start, end):
+            yield EnumDefinition(source_code,
+                                 span=match.span(),
+                                 group=match.groupdict())
+
+    def __init__(self, source_code, span=None, group=None):
+        assert isinstance(group, dict)
+        prefix_name = group.get('prefix_name', None)
+        suffix_name = group.get('suffix_name', None)
+        body = group.get('body', None)
+        assert prefix_name or suffix_name
+        assert body
+        assert span
+        # If suffix_name exists, it is a typedef
+        self._prototype = suffix_name if suffix_name else 'enum ' + prefix_name
+        self._name = suffix_name if suffix_name else prefix_name
+        self._body = body
+        self._source = source_code
+        self._span = span
+
+    def __repr__(self):
+        return 'Enum({},{})'.format(self._name, self._span)
+
+    def __str__(self):
+        return repr(self)
+
+    def span(self):
+        return self._span
+
+    def generate_tranlation_function(self):
+        """
+            Generate function for translating value to string
+        """
+        translation_table = []
+
+        for line in self._body.splitlines():
+
+            if line.strip().startswith('#'):
+                # Preprocess directive, keep it in table
+                translation_table.append(line.strip())
+                continue
+
+            if not line.strip():
+                continue
+
+            for field in line.strip().split(','):
+                if not field.strip():
+                    continue
+                member = field.strip().split()[0]
+                translation_table.append(
+                    '{space}[{member}] = "{member}",'.format(member=member,
+                                                             space=' '*8)
+                )
+
+        body = textwrap.dedent('''\
+            const char *{name}_str( {prototype} in )
+            {{
+                const char * in_to_str[]=
+                {{
+            {translation_table}
+                }};
+
+                if( in > ( sizeof( in_to_str )/sizeof( in_to_str[0]) - 1 ) ||
+                    in_to_str[ in ] == NULL )
+                {{
+                    return "UNKOWN_VAULE";
+                }}
+                return in_to_str[ in ];
+            }}
+                    ''')
+        body = body.format(translation_table='\n'.join(translation_table),
+                           name=self._name,
+                           prototype=self._prototype)
+        prototype = 'const char *{name}_str( {prototype} in );\n'
+        prototype = prototype.format(name=self._name,
+                                     prototype=self._prototype)
+        return body, prototype
+
+
+OUTPUT_C_TEMPLATE = '''\
+/* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */
+
+#include "common.h"
+
+#if defined(MBEDTLS_DEBUG_C)
+
+#include "ssl_debug_helpers_generated.h"
+
+{functions}
+
+#endif /* MBEDTLS_DEBUG_C */
+/* End of automatically generated file. */
+
+'''
+
+OUTPUT_H_TEMPLATE = '''\
+/* Automatically generated by generate_ssl_debug_helpers.py. DO NOT EDIT. */
+#ifndef MBEDTLS_SSL_DEBUG_HELPERS_H
+#define MBEDTLS_SSL_DEBUG_HELPERS_H
+
+#include "common.h"
+
+#if defined(MBEDTLS_DEBUG_C)
+
+#include "mbedtls/ssl.h"
+#include "ssl_misc.h"
+
+{functions}
+
+#endif /* MBEDTLS_DEBUG_C */
+
+#endif /* SSL_DEBUG_HELPERS_H */
+
+/* End of automatically generated file. */
+
+'''
+
+
+def generate_ssl_debug_helpers(output_directory, mbedtls_root):
+    """
+        Generate functions of debug helps
+    """
+    mbedtls_root = os.path.abspath(mbedtls_root or build_tree.guess_mbedtls_root())
+    with open(os.path.join(mbedtls_root, 'include/mbedtls/ssl.h')) as f:
+        source_code = remove_c_comments(f.read())
+
+    definitions = dict()
+    prototypes = dict()
+    for start, instance in preprocess_c_source_code(source_code, EnumDefinition):
+        if start in definitions:
+            continue
+        if isinstance(instance, EnumDefinition):
+            definition, prototype = instance.generate_tranlation_function()
+        else:
+            definition = instance
+            prototype = instance
+        definitions[start] = definition
+        prototypes[start] = prototype
+
+    function_definitions = [str(v) for _, v in sorted(definitions.items())]
+    function_prototypes = [str(v) for _, v in sorted(prototypes.items())]
+    if output_directory == sys.stdout:
+        sys.stdout.write(OUTPUT_H_TEMPLATE.format(
+            functions='\n'.join(function_prototypes)))
+        sys.stdout.write(OUTPUT_C_TEMPLATE.format(
+            functions='\n'.join(function_definitions)))
+    else:
+        with open(os.path.join(output_directory, 'ssl_debug_helpers_generated.c'), 'w') as f:
+            f.write(OUTPUT_C_TEMPLATE.format(
+                functions='\n'.join(function_definitions)))
+
+        with open(os.path.join(output_directory, 'ssl_debug_helpers_generated.h'), 'w') as f:
+            f.write(OUTPUT_H_TEMPLATE.format(
+                functions='\n'.join(function_prototypes)))
+
+
+def main():
+    """
+    Command line entry
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--mbedtls-root', nargs='?', default=None,
+                        help='root directory of mbedtls source code')
+    parser.add_argument('output_directory', nargs='?',
+                        default='library', help='source/header files location')
+
+    args = parser.parse_args()
+
+    generate_ssl_debug_helpers(args.output_directory, args.mbedtls_root)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/make_generated_files.bat

@@ -4,6 +4,7 @@
 perl scripts\generate_errors.pl || exit /b 1
 perl scripts\generate_query_config.pl || exit /b 1
 perl scripts\generate_features.pl || exit /b 1
+python scripts\generate_ssl_debug_helpers.py || exit /b 1
 perl scripts\generate_visualc_files.pl || exit /b 1
 python scripts\generate_psa_constants.py || exit /b 1
 python tests\scripts\generate_psa_tests.py || exit /b 1

scripts/mbedtls_dev/build_tree.py

@@ -17,12 +17,15 @@
 # limitations under the License.
 
 import os
+import inspect
+
 
 def looks_like_mbedtls_root(path: str) -> bool:
     """Whether the given directory looks like the root of the Mbed TLS source tree."""
     return all(os.path.isdir(os.path.join(path, subdir))
                for subdir in ['include', 'library', 'programs', 'tests'])
 
+
 def chdir_to_root() -> None:
     """Detect the root of the Mbed TLS source tree and change to it.
 
@@ -36,3 +39,22 @@ def chdir_to_root() -> None:
             os.chdir(d)
             return
     raise Exception('Mbed TLS source tree not found')
+
+
+def guess_mbedtls_root():
+    """Guess mbedTLS source code directory.
+
+    Return the first possible mbedTLS root directory
+    """
+    dirs = set({})
+    for frame in inspect.stack():
+        path = os.path.dirname(frame.filename)
+        for d in ['.', os.path.pardir] \
+                 + [os.path.join(*([os.path.pardir]*i)) for i in range(2, 10)]:
+            d = os.path.abspath(os.path.join(path, d))
+            if d in dirs:
+                continue
+            dirs.add(d)
+            if looks_like_mbedtls_root(d):
+                return d
+    raise Exception('Mbed TLS source tree not found')

tests/scripts/check-generated-files.sh

@@ -118,6 +118,7 @@ check()
 check scripts/generate_errors.pl library/error.c
 check scripts/generate_query_config.pl programs/test/query_config.c
 check scripts/generate_features.pl library/version_features.c
+check scripts/generate_ssl_debug_helpers.py library/ssl_debug_helpers_generated.c
 # generate_visualc_files enumerates source files (library/*.c). It doesn't
 # care about their content, but the files must exist. So it must run after
 # the step that creates or updates these files.

tests/ssl-opt.sh

@@ -8807,8 +8807,8 @@ run_test    "TLS 1.3: handshake dispatch test: tls13 only" \
             "$P_SRV debug_level=2 min_version=tls13 max_version=tls13" \
             "$P_CLI debug_level=2 min_version=tls13 max_version=tls13" \
             1 \
-            -s "tls13 server state: 0"     \
+            -s "tls13 server state: MBEDTLS_SSL_HELLO_REQUEST"     \
-            -c "tls13 client state: 0"
+            -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST"
 
 requires_openssl_tls1_3
 requires_config_enabled MBEDTLS_SSL_PROTO_TLS1_3_EXPERIMENTAL
@@ -8820,16 +8820,16 @@ run_test    "TLS 1.3: minimal feature sets - openssl" \
             "$O_NEXT_SRV -msg -tls1_3 -num_tickets 0 -no_resume_ephemeral -no_cache" \
             "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \
             0 \
-            -c "tls13 client state: 0"     \
+            -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)"               \
-            -c "tls13 client state: 2"     \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_HELLO(2)"                \
-            -c "tls13 client state: 19"    \
+            -c "tls13 client state: MBEDTLS_SSL_ENCRYPTED_EXTENSIONS(19)"       \
-            -c "tls13 client state: 5"     \
+            -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_REQUEST(5)"         \
-            -c "tls13 client state: 3"     \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_CERTIFICATE(3)"          \
-            -c "tls13 client state: 9"     \
+            -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_VERIFY(9)"          \
-            -c "tls13 client state: 13"    \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_FINISHED(13)"            \
-            -c "tls13 client state: 11"    \
+            -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)"            \
-            -c "tls13 client state: 14"    \
+            -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)"              \
-            -c "tls13 client state: 15"    \
+            -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)"           \
             -c "<= ssl_tls13_process_server_hello" \
             -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \
             -c "ECDH curve: x25519"         \
@@ -8854,16 +8854,16 @@ run_test    "TLS 1.3: minimal feature sets - gnutls" \
             "$P_CLI debug_level=3 min_version=tls13 max_version=tls13" \
             0 \
             -s "SERVER HELLO was queued"    \
-            -c "tls13 client state: 0"     \
+            -c "tls13 client state: MBEDTLS_SSL_HELLO_REQUEST(0)"               \
-            -c "tls13 client state: 2"     \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_HELLO(2)"                \
-            -c "tls13 client state: 19"    \
+            -c "tls13 client state: MBEDTLS_SSL_ENCRYPTED_EXTENSIONS(19)"       \
-            -c "tls13 client state: 5"     \
+            -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_REQUEST(5)"         \
-            -c "tls13 client state: 3"     \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_CERTIFICATE(3)"          \
-            -c "tls13 client state: 9"     \
+            -c "tls13 client state: MBEDTLS_SSL_CERTIFICATE_VERIFY(9)"          \
-            -c "tls13 client state: 13"    \
+            -c "tls13 client state: MBEDTLS_SSL_SERVER_FINISHED(13)"            \
-            -c "tls13 client state: 11"    \
+            -c "tls13 client state: MBEDTLS_SSL_CLIENT_FINISHED(11)"            \
-            -c "tls13 client state: 14"    \
+            -c "tls13 client state: MBEDTLS_SSL_FLUSH_BUFFERS(14)"              \
-            -c "tls13 client state: 15"    \
+            -c "tls13 client state: MBEDTLS_SSL_HANDSHAKE_WRAPUP(15)"           \
             -c "<= ssl_tls13_process_server_hello" \
             -c "server hello, chosen ciphersuite: ( 1301 ) - TLS1-3-AES-128-GCM-SHA256" \
             -c "ECDH curve: x25519"         \