Implement PMTU auto-reduction in handshake

ChangeLog

@@ -7,6 +7,9 @@ Features
      is controlled by the maximum fragment length as set locally or negotiated
      with the peer, as well as by a new per-connection MTU option, set using
      mbedtls_ssl_set_mtu().
+   * Add support for auto-adjustment of MTU to a safe value during the
+     handshake when flights do not get through (RFC 6347, section 4.1.1.1,
+     last paragraph).
 
 Bugfix
    * Fixes an issue with MBEDTLS_CHACHAPOLY_C which would not compile if

include/mbedtls/ssl_internal.h

@@ -307,6 +307,7 @@ struct mbedtls_ssl_handshake_params
                                               resending messages             */
     unsigned char alt_out_ctr[8];       /*!<  Alternative record epoch/counter
                                               for resending messages         */
+    uint16_t mtu;                       /*!<  Handshake mtu, used to fragment outoing messages */
 #endif /* MBEDTLS_SSL_PROTO_DTLS */
 
     /*

library/ssl_tls.c

@@ -108,6 +108,15 @@ static int ssl_double_retransmit_timeout( mbedtls_ssl_context *ssl )
     if( ssl->handshake->retransmit_timeout >= ssl->conf->hs_timeout_max )
         return( -1 );
 
+    /* Implement the final paragraph of RFC 6347 section 4.1.1.1
+     * in the following way: after the initial transmission and a first
+     * retransmission, back off to a temporary estimated MTU of 508 bytes.
+     * This value is guaranteed to be deliverable (if not guaranteed to be
+     * delivered) of any compliant IPv4 (and IPv6) network, and should work
+     * on most non-IP stacks too. */
+    if( ssl->handshake->retransmit_timeout != ssl->conf->hs_timeout_min )
+        ssl->handshake->mtu = 508;
+
     new_timeout = 2 * ssl->handshake->retransmit_timeout;
 
     /* Avoid arithmetic overflow and range overflow */
@@ -7088,6 +7097,20 @@ size_t mbedtls_ssl_get_max_frag_len( const mbedtls_ssl_context *ssl )
 }
 #endif /* MBEDTLS_SSL_MAX_FRAGMENT_LENGTH */
 
+#if defined(MBEDTLS_SSL_PROTO_DTLS)
+static size_t ssl_get_current_mtu( const mbedtls_ssl_context *ssl )
+{
+    if( ssl->handshake == NULL || ssl->handshake->mtu == 0 )
+        return( ssl->mtu );
+
+    if( ssl->mtu == 0 )
+        return( ssl->handshake->mtu );
+
+    return( ssl->mtu < ssl->handshake->mtu ?
+            ssl->mtu : ssl->handshake->mtu );
+}
+#endif /* MBEDTLS_SSL_PROTO_DTLS */
+
 int mbedtls_ssl_get_max_out_record_payload( const mbedtls_ssl_context *ssl )
 {
     size_t max_len = MBEDTLS_SSL_OUT_CONTENT_LEN;
@@ -7105,9 +7128,9 @@ int mbedtls_ssl_get_max_out_record_payload( const mbedtls_ssl_context *ssl )
 #endif
 
 #if defined(MBEDTLS_SSL_PROTO_DTLS)
-    if( ssl->mtu != 0 )
+    if( ssl_get_current_mtu( ssl ) != 0 )
     {
-        const size_t mtu = ssl->mtu;
+        const size_t mtu = ssl_get_current_mtu( ssl );
         const int ret = mbedtls_ssl_get_record_expansion( ssl );
         const size_t overhead = (size_t) ret;
 
@@ -7123,7 +7146,7 @@ int mbedtls_ssl_get_max_out_record_payload( const mbedtls_ssl_context *ssl )
         if( max_len > mtu - overhead )
             max_len = mtu - overhead;
     }
-#endif
+#endif /* MBEDTLS_SSL_PROTO_DTLS */
 
     return( (int) max_len );
 }

tests/ssl-opt.sh

@@ -5111,6 +5111,25 @@ run_test    "DTLS fragmenting: both (MTU)" \
             -c "found fragmented DTLS handshake message" \
             -C "error"
 
+# Test for automatic MTU reduction on repeated resend
+requires_config_enabled MBEDTLS_SSL_PROTO_DTLS
+requires_config_enabled MBEDTLS_RSA_C
+requires_config_enabled MBEDTLS_ECDSA_C
+run_test    "DTLS fragmenting: proxy MTU: auto-reduction" \
+            -p "$P_PXY mtu=508" \
+            "$P_SRV dtls=1 debug_level=2 auth_mode=required \
+             crt_file=data_files/server7_int-ca.crt \
+             key_file=data_files/server7.key\
+             hs_timeout=100-400" \
+            "$P_CLI dtls=1 debug_level=2 \
+             crt_file=data_files/server8_int-ca2.crt \
+             key_file=data_files/server8.key \
+             hs_timeout=100-400" \
+            0 \
+            -s "found fragmented DTLS handshake message" \
+            -c "found fragmented DTLS handshake message" \
+            -C "error"
+
 # the proxy shouldn't drop or mess up anything, so we shouldn't need to resend
 # OTOH the client might resend if the server is to slow to reset after sending
 # a HelloVerifyRequest, so only check for no retransmission server-side