Merge pull request #5189 from gilles-peskine-arm/struct_reordering_2.x

Backport 2.x: Reorder structure fields to maximize usage of immediate offset access
This commit is contained in:
Manuel Pégourié-Gonnard 2021-12-09 12:54:13 +01:00 committed by GitHub
commit c3319e73db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 338 additions and 257 deletions

View file

@ -0,0 +1,5 @@
API changes
* Some fields of mbedtls_ssl_session and mbedtls_ssl_config are in a
different order. This only affects applications that define such
structures directly or serialize them.

View file

@ -974,6 +974,10 @@ mbedtls_dtls_srtp_info;
*/
struct mbedtls_ssl_session
{
#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH)
unsigned char mfl_code; /*!< MaxFragmentLength negotiated by peer */
#endif /* MBEDTLS_SSL_MAX_FRAGMENT_LENGTH */
#if defined(MBEDTLS_HAVE_TIME)
mbedtls_time_t start; /*!< starting time */
#endif
@ -1002,10 +1006,6 @@ struct mbedtls_ssl_session
uint32_t ticket_lifetime; /*!< ticket lifetime hint */
#endif /* MBEDTLS_SSL_SESSION_TICKETS && MBEDTLS_SSL_CLI_C */
#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH)
unsigned char mfl_code; /*!< MaxFragmentLength negotiated by peer */
#endif /* MBEDTLS_SSL_MAX_FRAGMENT_LENGTH */
#if defined(MBEDTLS_SSL_TRUNCATED_HMAC)
int trunc_hmac; /*!< flag for truncated hmac activation */
#endif /* MBEDTLS_SSL_TRUNCATED_HMAC */
@ -1020,7 +1020,98 @@ struct mbedtls_ssl_session
*/
struct mbedtls_ssl_config
{
/* Group items by size (largest first) to minimize padding overhead */
/* Group items by size and reorder them to maximize usage of immediate offset access. */
/*
* Numerical settings (char)
*/
unsigned char max_major_ver; /*!< max. major version used */
unsigned char max_minor_ver; /*!< max. minor version used */
unsigned char min_major_ver; /*!< min. major version used */
unsigned char min_minor_ver; /*!< min. minor version used */
/*
* Flags (could be bit-fields to save RAM, but separate bytes make
* the code smaller on architectures with an instruction for direct
* byte access).
*/
uint8_t endpoint /*bool*/; /*!< 0: client, 1: server */
uint8_t transport /*bool*/; /*!< stream (TLS) or datagram (DTLS) */
uint8_t authmode /*2 bits*/; /*!< MBEDTLS_SSL_VERIFY_XXX */
/* needed even with renego disabled for LEGACY_BREAK_HANDSHAKE */
uint8_t allow_legacy_renegotiation /*2 bits*/; /*!< MBEDTLS_LEGACY_XXX */
#if defined(MBEDTLS_ARC4_C)
uint8_t arc4_disabled /*bool*/; /*!< blacklist RC4 ciphersuites? */
#endif
#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH)
uint8_t mfl_code /*3 bits*/; /*!< desired fragment length */
#endif
#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC)
uint8_t encrypt_then_mac /*bool*/; /*!< negotiate encrypt-then-mac? */
#endif
#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET)
uint8_t extended_ms /*bool*/; /*!< negotiate extended master secret? */
#endif
#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY)
uint8_t anti_replay /*bool*/; /*!< detect and prevent replay? */
#endif
#if defined(MBEDTLS_SSL_CBC_RECORD_SPLITTING)
uint8_t cbc_record_splitting /*bool*/; /*!< do cbc record splitting */
#endif
#if defined(MBEDTLS_SSL_RENEGOTIATION)
uint8_t disable_renegotiation /*bool*/; /*!< disable renegotiation? */
#endif
#if defined(MBEDTLS_SSL_TRUNCATED_HMAC)
uint8_t trunc_hmac /*bool*/; /*!< negotiate truncated hmac? */
#endif
#if defined(MBEDTLS_SSL_SESSION_TICKETS)
uint8_t session_tickets /*bool*/; /*!< use session tickets? */
#endif
#if defined(MBEDTLS_SSL_FALLBACK_SCSV) && defined(MBEDTLS_SSL_CLI_C)
uint8_t fallback /*bool*/; /*!< is this a fallback? */
#endif
#if defined(MBEDTLS_SSL_SRV_C)
uint8_t cert_req_ca_list /*bool*/; /*!< enable sending CA list in
Certificate Request messages? */
#endif
#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID)
uint8_t ignore_unexpected_cid /*bool*/; /*!< Determines whether DTLS
* record with unexpected CID
* should lead to failure. */
#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */
#if defined(MBEDTLS_SSL_DTLS_SRTP)
uint8_t dtls_srtp_mki_support /*bool*/; /*!< support having mki_value
in the use_srtp extension? */
#endif
/*
* Numerical settings (int or larger)
*/
uint32_t read_timeout; /*!< timeout for mbedtls_ssl_read (ms) */
#if defined(MBEDTLS_SSL_PROTO_DTLS)
uint32_t hs_timeout_min; /*!< initial value of the handshake
retransmission timeout (ms) */
uint32_t hs_timeout_max; /*!< maximum value of the handshake
retransmission timeout (ms) */
#endif
#if defined(MBEDTLS_SSL_RENEGOTIATION)
int renego_max_records; /*!< grace period for renegotiation */
unsigned char renego_period[8]; /*!< value of the record counters
that triggers renegotiation */
#endif
#if defined(MBEDTLS_SSL_DTLS_BADMAC_LIMIT)
unsigned int badmac_limit; /*!< limit of records with a bad MAC */
#endif
#if defined(MBEDTLS_DHM_C) && defined(MBEDTLS_SSL_CLI_C)
unsigned int dhm_min_bitlen; /*!< min. bit length of the DHM prime */
#endif
/*
* Pointers
@ -1174,91 +1265,6 @@ struct mbedtls_ssl_config
/*! number of supported profiles */
size_t dtls_srtp_profile_list_len;
#endif /* MBEDTLS_SSL_DTLS_SRTP */
/*
* Numerical settings (int then char)
*/
uint32_t read_timeout; /*!< timeout for mbedtls_ssl_read (ms) */
#if defined(MBEDTLS_SSL_PROTO_DTLS)
uint32_t hs_timeout_min; /*!< initial value of the handshake
retransmission timeout (ms) */
uint32_t hs_timeout_max; /*!< maximum value of the handshake
retransmission timeout (ms) */
#endif
#if defined(MBEDTLS_SSL_RENEGOTIATION)
int renego_max_records; /*!< grace period for renegotiation */
unsigned char renego_period[8]; /*!< value of the record counters
that triggers renegotiation */
#endif
#if defined(MBEDTLS_SSL_DTLS_BADMAC_LIMIT)
unsigned int badmac_limit; /*!< limit of records with a bad MAC */
#endif
#if defined(MBEDTLS_DHM_C) && defined(MBEDTLS_SSL_CLI_C)
unsigned int dhm_min_bitlen; /*!< min. bit length of the DHM prime */
#endif
unsigned char max_major_ver; /*!< max. major version used */
unsigned char max_minor_ver; /*!< max. minor version used */
unsigned char min_major_ver; /*!< min. major version used */
unsigned char min_minor_ver; /*!< min. minor version used */
/*
* Flags (bitfields)
*/
unsigned int endpoint : 1; /*!< 0: client, 1: server */
unsigned int transport : 1; /*!< stream (TLS) or datagram (DTLS) */
unsigned int authmode : 2; /*!< MBEDTLS_SSL_VERIFY_XXX */
/* needed even with renego disabled for LEGACY_BREAK_HANDSHAKE */
unsigned int allow_legacy_renegotiation : 2 ; /*!< MBEDTLS_LEGACY_XXX */
#if defined(MBEDTLS_ARC4_C)
unsigned int arc4_disabled : 1; /*!< blacklist RC4 ciphersuites? */
#endif
#if defined(MBEDTLS_SSL_MAX_FRAGMENT_LENGTH)
unsigned int mfl_code : 3; /*!< desired fragment length */
#endif
#if defined(MBEDTLS_SSL_ENCRYPT_THEN_MAC)
unsigned int encrypt_then_mac : 1 ; /*!< negotiate encrypt-then-mac? */
#endif
#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET)
unsigned int extended_ms : 1; /*!< negotiate extended master secret? */
#endif
#if defined(MBEDTLS_SSL_DTLS_ANTI_REPLAY)
unsigned int anti_replay : 1; /*!< detect and prevent replay? */
#endif
#if defined(MBEDTLS_SSL_CBC_RECORD_SPLITTING)
unsigned int cbc_record_splitting : 1; /*!< do cbc record splitting */
#endif
#if defined(MBEDTLS_SSL_RENEGOTIATION)
unsigned int disable_renegotiation : 1; /*!< disable renegotiation? */
#endif
#if defined(MBEDTLS_SSL_TRUNCATED_HMAC)
unsigned int trunc_hmac : 1; /*!< negotiate truncated hmac? */
#endif
#if defined(MBEDTLS_SSL_SESSION_TICKETS)
unsigned int session_tickets : 1; /*!< use session tickets? */
#endif
#if defined(MBEDTLS_SSL_FALLBACK_SCSV) && defined(MBEDTLS_SSL_CLI_C)
unsigned int fallback : 1; /*!< is this a fallback? */
#endif
#if defined(MBEDTLS_SSL_SRV_C)
unsigned int cert_req_ca_list : 1; /*!< enable sending CA list in
Certificate Request messages? */
#endif
#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID)
unsigned int ignore_unexpected_cid : 1; /*!< Determines whether DTLS
* record with unexpected CID
* should lead to failure. */
#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */
#if defined(MBEDTLS_SSL_DTLS_SRTP)
unsigned int dtls_srtp_mki_support : 1; /* support having mki_value
in the use_srtp extension */
#endif
};
struct mbedtls_ssl_context

View file

@ -430,13 +430,63 @@ struct mbedtls_ssl_handshake_params
* Handshake specific crypto variables
*/
uint8_t max_major_ver; /*!< max. major version client*/
uint8_t max_minor_ver; /*!< max. minor version client*/
uint8_t resume; /*!< session resume indicator*/
uint8_t cli_exts; /*!< client extension presence*/
#if defined(MBEDTLS_X509_CRT_PARSE_C) && \
defined(MBEDTLS_SSL_SERVER_NAME_INDICATION)
uint8_t sni_authmode; /*!< authmode from SNI callback */
#endif
#if defined(MBEDTLS_SSL_SESSION_TICKETS)
uint8_t new_session_ticket; /*!< use NewSessionTicket? */
#endif /* MBEDTLS_SSL_SESSION_TICKETS */
#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET)
uint8_t extended_ms; /*!< use Extended Master Secret? */
#endif
#if defined(MBEDTLS_SSL_ASYNC_PRIVATE)
uint8_t async_in_progress; /*!< an asynchronous operation is in progress */
#endif /* MBEDTLS_SSL_ASYNC_PRIVATE */
#if defined(MBEDTLS_SSL_PROTO_DTLS)
unsigned char retransmit_state; /*!< Retransmission state */
#endif
#if defined(MBEDTLS_SSL_ECP_RESTARTABLE_ENABLED)
uint8_t ecrs_enabled; /*!< Handshake supports EC restart? */
enum { /* this complements ssl->state with info on intra-state operations */
ssl_ecrs_none = 0, /*!< nothing going on (yet) */
ssl_ecrs_crt_verify, /*!< Certificate: crt_verify() */
ssl_ecrs_ske_start_processing, /*!< ServerKeyExchange: pk_verify() */
ssl_ecrs_cke_ecdh_calc_secret, /*!< ClientKeyExchange: ECDH step 2 */
ssl_ecrs_crt_vrfy_sign, /*!< CertificateVerify: pk_sign() */
} ecrs_state; /*!< current (or last) operation */
mbedtls_x509_crt *ecrs_peer_cert; /*!< The peer's CRT chain. */
size_t ecrs_n; /*!< place for saving a length */
#endif
#if defined(MBEDTLS_SSL_PROTO_TLS1_2) && \
defined(MBEDTLS_KEY_EXCHANGE_WITH_CERT_ENABLED)
mbedtls_ssl_sig_hash_set_t hash_algs; /*!< Set of suitable sig-hash pairs */
#endif
size_t pmslen; /*!< premaster length */
mbedtls_ssl_ciphersuite_t const *ciphersuite_info;
void (*update_checksum)(mbedtls_ssl_context *, const unsigned char *, size_t);
void (*calc_verify)(const mbedtls_ssl_context *, unsigned char *, size_t *);
void (*calc_finished)(mbedtls_ssl_context *, unsigned char *, int);
mbedtls_ssl_tls_prf_cb *tls_prf;
#if defined(MBEDTLS_DHM_C)
mbedtls_dhm_context dhm_ctx; /*!< DHM key exchange */
#endif
/* Adding guard for MBEDTLS_ECDSA_C to ensure no compile errors due
* to guards also being in ssl_srv.c and ssl_cli.c. There is a gap
* in functionality that access to ecdh_ctx structure is needed for
@ -461,10 +511,12 @@ struct mbedtls_ssl_handshake_params
size_t ecjpake_cache_len; /*!< Length of cached data */
#endif
#endif /* MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED */
#if defined(MBEDTLS_ECDH_C) || defined(MBEDTLS_ECDSA_C) || \
defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED)
const mbedtls_ecp_curve_info **curves; /*!< Supported elliptic curves */
#endif
#if defined(MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED)
#if defined(MBEDTLS_USE_PSA_CRYPTO)
psa_key_id_t psk_opaque; /*!< Opaque PSK from the callback */
@ -472,65 +524,26 @@ struct mbedtls_ssl_handshake_params
unsigned char *psk; /*!< PSK from the callback */
size_t psk_len; /*!< Length of PSK from callback */
#endif /* MBEDTLS_KEY_EXCHANGE_SOME_PSK_ENABLED */
#if defined(MBEDTLS_X509_CRT_PARSE_C)
mbedtls_ssl_key_cert *key_cert; /*!< chosen key/cert pair (server) */
#if defined(MBEDTLS_SSL_SERVER_NAME_INDICATION)
int sni_authmode; /*!< authmode from SNI callback */
mbedtls_ssl_key_cert *sni_key_cert; /*!< key/cert list from SNI */
mbedtls_x509_crt *sni_ca_chain; /*!< trusted CAs from SNI callback */
mbedtls_x509_crl *sni_ca_crl; /*!< trusted CAs CRLs from SNI */
#endif /* MBEDTLS_SSL_SERVER_NAME_INDICATION */
#endif /* MBEDTLS_X509_CRT_PARSE_C */
#if defined(MBEDTLS_SSL_ECP_RESTARTABLE_ENABLED)
int ecrs_enabled; /*!< Handshake supports EC restart? */
mbedtls_x509_crt_restart_ctx ecrs_ctx; /*!< restart context */
enum { /* this complements ssl->state with info on intra-state operations */
ssl_ecrs_none = 0, /*!< nothing going on (yet) */
ssl_ecrs_crt_verify, /*!< Certificate: crt_verify() */
ssl_ecrs_ske_start_processing, /*!< ServerKeyExchange: pk_verify() */
ssl_ecrs_cke_ecdh_calc_secret, /*!< ClientKeyExchange: ECDH step 2 */
ssl_ecrs_crt_vrfy_sign, /*!< CertificateVerify: pk_sign() */
} ecrs_state; /*!< current (or last) operation */
mbedtls_x509_crt *ecrs_peer_cert; /*!< The peer's CRT chain. */
size_t ecrs_n; /*!< place for saving a length */
#endif
#if defined(MBEDTLS_X509_CRT_PARSE_C) && \
!defined(MBEDTLS_SSL_KEEP_PEER_CERTIFICATE)
mbedtls_pk_context peer_pubkey; /*!< The public key from the peer. */
#endif /* MBEDTLS_X509_CRT_PARSE_C && !MBEDTLS_SSL_KEEP_PEER_CERTIFICATE */
#if defined(MBEDTLS_SSL_PROTO_DTLS)
unsigned int out_msg_seq; /*!< Outgoing handshake sequence number */
unsigned int in_msg_seq; /*!< Incoming handshake sequence number */
unsigned char *verify_cookie; /*!< Cli: HelloVerifyRequest cookie
Srv: unused */
unsigned char verify_cookie_len; /*!< Cli: cookie length
Srv: flag for sending a cookie */
uint32_t retransmit_timeout; /*!< Current value of timeout */
unsigned char retransmit_state; /*!< Retransmission state */
mbedtls_ssl_flight_item *flight; /*!< Current outgoing flight */
mbedtls_ssl_flight_item *cur_msg; /*!< Current message in flight */
unsigned char *cur_msg_p; /*!< Position in current message */
unsigned int in_flight_start_seq; /*!< Minimum message sequence in the
flight being received */
mbedtls_ssl_transform *alt_transform_out; /*!< Alternative transform for
resending messages */
unsigned char alt_out_ctr[8]; /*!< Alternative record epoch/counter
for resending messages */
#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID)
/* The state of CID configuration in this handshake. */
uint8_t cid_in_use; /*!< This indicates whether the use of the CID extension
* has been negotiated. Possible values are
* #MBEDTLS_SSL_CID_ENABLED and
* #MBEDTLS_SSL_CID_DISABLED. */
unsigned char peer_cid[ MBEDTLS_SSL_CID_OUT_LEN_MAX ]; /*! The peer's CID */
uint8_t peer_cid_len; /*!< The length of
* \c peer_cid. */
#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */
struct
{
size_t total_bytes_buffered; /*!< Cumulative size of heap allocated
@ -557,6 +570,37 @@ struct mbedtls_ssl_handshake_params
} buffering;
unsigned int out_msg_seq; /*!< Outgoing handshake sequence number */
unsigned int in_msg_seq; /*!< Incoming handshake sequence number */
unsigned char *verify_cookie; /*!< Cli: HelloVerifyRequest cookie
Srv: unused */
unsigned char verify_cookie_len; /*!< Cli: cookie length
Srv: flag for sending a cookie */
uint32_t retransmit_timeout; /*!< Current value of timeout */
mbedtls_ssl_flight_item *flight; /*!< Current outgoing flight */
mbedtls_ssl_flight_item *cur_msg; /*!< Current message in flight */
unsigned char *cur_msg_p; /*!< Position in current message */
unsigned int in_flight_start_seq; /*!< Minimum message sequence in the
flight being received */
mbedtls_ssl_transform *alt_transform_out; /*!< Alternative transform for
resending messages */
unsigned char alt_out_ctr[8]; /*!< Alternative record epoch/counter
for resending messages */
#if defined(MBEDTLS_SSL_DTLS_CONNECTION_ID)
/* The state of CID configuration in this handshake. */
uint8_t cid_in_use; /*!< This indicates whether the use of the CID extension
* has been negotiated. Possible values are
* #MBEDTLS_SSL_CID_ENABLED and
* #MBEDTLS_SSL_CID_DISABLED. */
unsigned char peer_cid[ MBEDTLS_SSL_CID_OUT_LEN_MAX ]; /*! The peer's CID */
uint8_t peer_cid_len; /*!< The length of
* \c peer_cid. */
#endif /* MBEDTLS_SSL_DTLS_CONNECTION_ID */
uint16_t mtu; /*!< Handshake mtu, used to fragment outgoing messages */
#endif /* MBEDTLS_SSL_PROTO_DTLS */
@ -585,35 +629,10 @@ struct mbedtls_ssl_handshake_params
#endif
#endif /* MBEDTLS_SSL_PROTO_TLS1_2 */
void (*update_checksum)(mbedtls_ssl_context *, const unsigned char *, size_t);
void (*calc_verify)(const mbedtls_ssl_context *, unsigned char *, size_t *);
void (*calc_finished)(mbedtls_ssl_context *, unsigned char *, int);
mbedtls_ssl_tls_prf_cb *tls_prf;
mbedtls_ssl_ciphersuite_t const *ciphersuite_info;
size_t pmslen; /*!< premaster length */
unsigned char randbytes[64]; /*!< random bytes */
unsigned char premaster[MBEDTLS_PREMASTER_SIZE];
/*!< premaster secret */
int resume; /*!< session resume indicator*/
int max_major_ver; /*!< max. major version client*/
int max_minor_ver; /*!< max. minor version client*/
int cli_exts; /*!< client extension presence*/
#if defined(MBEDTLS_SSL_SESSION_TICKETS)
int new_session_ticket; /*!< use NewSessionTicket? */
#endif /* MBEDTLS_SSL_SESSION_TICKETS */
#if defined(MBEDTLS_SSL_EXTENDED_MASTER_SECRET)
int extended_ms; /*!< use Extended Master Secret? */
#endif
#if defined(MBEDTLS_SSL_ASYNC_PRIVATE)
unsigned int async_in_progress : 1; /*!< an asynchronous operation is in progress */
#endif /* MBEDTLS_SSL_ASYNC_PRIVATE */
#if defined(MBEDTLS_SSL_ASYNC_PRIVATE)
/** Asynchronous operation context. This field is meant for use by the
* asynchronous operation callbacks (mbedtls_ssl_config::f_async_sign_start,

View file

@ -42,6 +42,9 @@
* of these types. */
#include "crypto_types.h"
#include "crypto_values.h"
/* Include size definitions which are used to size some arrays in operation
* structures. */
#include <psa/crypto_sizes.h>
/** For encrypt-decrypt functions, whether the operation is an encryption
* or a decryption. */

View file

@ -30,6 +30,7 @@
#include "mbedtls/platform_util.h"
#include "crypto_types.h"
#include "crypto_compat.h"
#ifdef __cplusplus

View file

@ -175,9 +175,6 @@ typedef struct
{
uint8_t *info;
size_t info_length;
psa_mac_operation_t hmac;
uint8_t prk[PSA_HASH_MAX_SIZE];
uint8_t output_block[PSA_HASH_MAX_SIZE];
#if PSA_HASH_MAX_SIZE > 0xff
#error "PSA_HASH_MAX_SIZE does not fit in uint8_t"
#endif
@ -185,6 +182,9 @@ typedef struct
uint8_t block_number;
unsigned int state : 2;
unsigned int info_set : 1;
uint8_t output_block[PSA_HASH_MAX_SIZE];
uint8_t prk[PSA_HASH_MAX_SIZE];
struct psa_mac_operation_s hmac;
} psa_hkdf_key_derivation_t;
#endif /* MBEDTLS_PSA_BUILTIN_ALG_HKDF */

View file

@ -108,9 +108,9 @@ static int key_type_is_raw_bytes( psa_key_type_t type )
typedef struct
{
mbedtls_psa_random_context_t rng;
unsigned initialized : 1;
unsigned rng_state : 2;
mbedtls_psa_random_context_t rng;
} psa_global_data_t;
static psa_global_data_t global_data;

View file

@ -32,6 +32,8 @@
typedef struct
{
psa_algorithm_t core_alg;
uint8_t tag_length;
union
{
unsigned dummy; /* Make the union non-empty even with no supported algorithms. */
@ -45,11 +47,9 @@ typedef struct
mbedtls_chachapoly_context chachapoly;
#endif /* MBEDTLS_PSA_BUILTIN_ALG_CHACHA20_POLY1305 */
} ctx;
psa_algorithm_t core_alg;
uint8_t tag_length;
} aead_operation_t;
#define AEAD_OPERATION_INIT {{0}, 0, 0}
#define AEAD_OPERATION_INIT {0, 0, {0}}
static void psa_aead_abort_internal( aead_operation_t *operation )
{

View file

@ -457,50 +457,87 @@ class CodeParser():
return enum_consts
def parse_identifiers(self, include, exclude=None):
"""
Parse all lines of a header where a function/enum/struct/union/typedef
identifier is declared, based on some regex and heuristics. Highly
dependent on formatting style.
IGNORED_CHUNK_REGEX = re.compile('|'.join([
r'/\*.*?\*/', # block comment entirely on one line
r'//.*', # line comment
r'(?P<string>")(?:[^\\\"]|\\.)*"', # string literal
]))
Args:
* include: A List of glob expressions to look for files through.
* exclude: A List of glob expressions for excluding files.
def strip_comments_and_literals(self, line, in_block_comment):
"""Strip comments and string literals from line.
Returns a List of Match objects with identifiers.
Continuation lines are not supported.
If in_block_comment is true, assume that the line starts inside a
block comment.
Return updated values of (line, in_block_comment) where:
* Comments in line have been replaced by a space (or nothing at the
start or end of the line).
* String contents have been removed.
* in_block_comment indicates whether the line ends inside a block
comment that continues on the next line.
"""
identifier_regex = re.compile(
# Terminate current multiline comment?
if in_block_comment:
m = re.search(r"\*/", line)
if m:
in_block_comment = False
line = line[m.end(0):]
else:
return '', True
# Remove full comments and string literals.
# Do it all together to handle cases like "/*" correctly.
# Note that continuation lines are not supported.
line = re.sub(self.IGNORED_CHUNK_REGEX,
lambda s: '""' if s.group('string') else ' ',
line)
# Start an unfinished comment?
# (If `/*` was part of a complete comment, it's already been removed.)
m = re.search(r"/\*", line)
if m:
in_block_comment = True
line = line[:m.start(0)]
return line, in_block_comment
IDENTIFIER_REGEX = re.compile('|'.join([
# Match " something(a" or " *something(a". Functions.
# Assumptions:
# - function definition from return type to one of its arguments is
# all on one line
# - function definition line only contains alphanumeric, asterisk,
# underscore, and open bracket
r".* \**(\w+) *\( *\w|"
r".* \**(\w+) *\( *\w",
# Match "(*something)(".
r".*\( *\* *(\w+) *\) *\(|"
r".*\( *\* *(\w+) *\) *\(",
# Match names of named data structures.
r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$|"
r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$",
# Match names of typedef instances, after closing bracket.
r"}? *(\w+)[;[].*"
)
r"}? *(\w+)[;[].*",
]))
# The regex below is indented for clarity.
exclusion_lines = re.compile(
r"^("
r"extern +\"C\"|" # pylint: disable=bad-continuation
r"(typedef +)?(struct|union|enum)( *{)?$|"
r"} *;?$|"
r"$|"
r"//|"
r"#"
r")"
)
EXCLUSION_LINES = re.compile("|".join([
r"extern +\"C\"",
r"(typedef +)?(struct|union|enum)( *{)?$",
r"} *;?$",
r"$",
r"//",
r"#",
]))
files = self.get_files(include, exclude)
self.log.debug("Looking for identifiers in {} files".format(len(files)))
def parse_identifiers_in_file(self, header_file, identifiers):
"""
Parse all lines of a header where a function/enum/struct/union/typedef
identifier is declared, based on some regex and heuristics. Highly
dependent on formatting style.
Append found matches to the list ``identifiers``.
"""
identifiers = []
for header_file in files:
with open(header_file, "r", encoding="utf-8") as header:
in_block_comment = False
# The previous line variable is used for concatenating lines
@ -509,25 +546,15 @@ class CodeParser():
previous_line = ""
for line_no, line in enumerate(header):
# Skip parsing this line if a block comment ends on it,
# but don't skip if it has just started -- there is a chance
# it ends on the same line.
if re.search(r"/\*", line):
in_block_comment = not in_block_comment
if re.search(r"\*/", line):
in_block_comment = not in_block_comment
continue
line, in_block_comment = \
self.strip_comments_and_literals(line, in_block_comment)
if in_block_comment:
previous_line = ""
continue
if exclusion_lines.search(line):
if self.EXCLUSION_LINES.match(line):
previous_line = ""
continue
# If the line contains only space-separated alphanumeric
# characters (or underscore, asterisk, or, open bracket),
# characters (or underscore, asterisk, or open parenthesis),
# and nothing else, high chance it's a declaration that
# continues on the next line
if re.search(r"^([\w\*\(]+\s+)+$", line):
@ -546,7 +573,7 @@ class CodeParser():
if line[0] == " ":
continue
identifier = identifier_regex.search(line)
identifier = self.IDENTIFIER_REGEX.search(line)
if not identifier:
continue
@ -563,6 +590,26 @@ class CodeParser():
identifier.span(),
group))
def parse_identifiers(self, include, exclude=None):
"""
Parse all lines of a header where a function/enum/struct/union/typedef
identifier is declared, based on some regex and heuristics. Highly
dependent on formatting style.
Args:
* include: A List of glob expressions to look for files through.
* exclude: A List of glob expressions for excluding files.
Returns a List of Match objects with identifiers.
"""
files = self.get_files(include, exclude)
self.log.debug("Looking for identifiers in {} files".format(len(files)))
identifiers = []
for header_file in files:
self.parse_identifiers_in_file(header_file, identifiers)
return identifiers
def parse_symbols(self):