Merge pull request #5097 from gilles-peskine-arm/ssl-opt-resend-retry-2.x

Backport 2.x: Retry a test case if it fails due to an unexpected resend
This commit is contained in:
paul-elliott-arm 2021-10-27 16:24:41 +01:00 committed by GitHub
commit 460d779773
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -485,6 +485,32 @@ record_outcome() {
fi
}
# True if the presence of the given pattern in a log definitely indicates
# that the test has failed. False if the presence is inconclusive.
#
# Inputs:
# * $1: pattern found in the logs
# * $TIMES_LEFT: >0 if retrying is an option
#
# Outputs:
# * $outcome: set to a retry reason if the pattern is inconclusive,
# unchanged otherwise.
# * Return value: 1 if the pattern is inconclusive,
# 0 if the failure is definitive.
log_pattern_presence_is_conclusive() {
# If we've run out of attempts, then don't retry no matter what.
if [ $TIMES_LEFT -eq 0 ]; then
return 0
fi
case $1 in
"resend")
# An undesired resend may have been caused by the OS dropping or
# delaying a packet at an inopportune time.
outcome="RETRY(resend)"
return 1;;
esac
}
# fail <message>
fail() {
record_outcome "FAIL" "$1"
@ -733,68 +759,12 @@ find_in_both() {
fi
}
# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
# Options: -s pattern pattern that must be present in server output
# -c pattern pattern that must be present in client output
# -u pattern lines after pattern must be unique in client output
# -f call shell function on client output
# -S pattern pattern that must be absent in server output
# -C pattern pattern that must be absent in client output
# -U pattern lines after pattern must be unique in server output
# -F call shell function on server output
# -g call shell function on server and client output
run_test() {
NAME="$1"
shift 1
if is_excluded "$NAME"; then
SKIP_NEXT="NO"
# There was no request to run the test, so don't record its outcome.
return
fi
print_name "$NAME"
# Do we only run numbered tests?
if [ -n "$RUN_TEST_NUMBER" ]; then
case ",$RUN_TEST_NUMBER," in
*",$TESTS,"*) :;;
*) SKIP_NEXT="YES";;
esac
fi
# does this test use a proxy?
if [ "X$1" = "X-p" ]; then
PXY_CMD="$2"
shift 2
else
PXY_CMD=""
fi
# get commands and client output
SRV_CMD="$1"
CLI_CMD="$2"
CLI_EXPECT="$3"
shift 3
# Check if test uses files
case "$SRV_CMD $CLI_CMD" in
*data_files/*)
requires_config_enabled MBEDTLS_FS_IO;;
esac
# If the client or serve requires a ciphersuite, check that it's enabled.
maybe_requires_ciphersuite_enabled "$SRV_CMD" "$@"
maybe_requires_ciphersuite_enabled "$CLI_CMD" "$@"
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
record_outcome "SKIP"
SKIPS=$(( $SKIPS + 1 ))
return
fi
# Analyze the commands that will be used in a test.
#
# Analyze and possibly instrument $PXY_CMD, $CLI_CMD, $SRV_CMD to pass
# extra arguments or go through wrappers.
# Set $DTLS (0=TLS, 1=DTLS).
analyze_test_commands() {
# update DTLS variable
detect_dtls "$SRV_CMD"
@ -848,49 +818,30 @@ run_test() {
CLI_CMD="valgrind --leak-check=full $CLI_CMD"
fi
fi
}
TIMES_LEFT=2
while [ $TIMES_LEFT -gt 0 ]; do
TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
# Check for failure conditions after a test case.
#
# Inputs from run_test:
# * positional parameters: test options (see run_test documentation)
# * $CLI_EXIT: client return code
# * $CLI_EXPECT: expected client return code
# * $SRV_RET: server return code
# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files containing client/server/proxy logs
# * $TIMES_LEFT: if nonzero, a RETRY outcome is allowed
#
# Outputs:
# * $outcome: one of PASS/RETRY*/FAIL
check_test_failure() {
outcome=FAIL
# run the commands
if [ -n "$PXY_CMD" ]; then
printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
$PXY_CMD >> $PXY_OUT 2>&1 &
PXY_PID=$!
wait_proxy_start "$PXY_PORT" "$PXY_PID"
if [ $TIMES_LEFT -gt 0 ] &&
grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null
then
outcome="RETRY(client-timeout)"
return
fi
check_osrv_dtls
printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
SRV_PID=$!
wait_server_start "$SRV_PORT" "$SRV_PID"
printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
wait_client_done
sleep 0.05
# terminate the server (and the proxy)
kill $SRV_PID
wait $SRV_PID
SRV_RET=$?
if [ -n "$PXY_CMD" ]; then
kill $PXY_PID >/dev/null 2>&1
wait $PXY_PID
fi
# retry only on timeouts
if grep '===CLIENT_TIMEOUT===' $CLI_OUT >/dev/null; then
printf "RETRY "
else
TIMES_LEFT=0
fi
done
# check if the client and server went at least to the handshake stage
# (useful to avoid tests with only negative assertions and non-zero
# expected client exit to incorrectly succeed in case of catastrophic
@ -948,14 +899,18 @@ run_test() {
"-S")
if grep -v '^==' $SRV_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
if log_pattern_presence_is_conclusive "$2"; then
fail "pattern '$2' MUST NOT be present in the Server output"
fi
return
fi
;;
"-C")
if grep -v '^==' $CLI_OUT | grep -v 'Serious error when reading debug info' | grep "$2" >/dev/null; then
if log_pattern_presence_is_conclusive "$2"; then
fail "pattern '$2' MUST NOT be present in the Client output"
fi
return
fi
;;
@ -1019,6 +974,131 @@ run_test() {
fi
# if we're here, everything is ok
outcome=PASS
}
# Run the current test case: start the server and if applicable the proxy, run
# the client, wait for all processes to finish or time out.
#
# Inputs:
# * $NAME: test case name
# * $CLI_CMD, $SRV_CMD, $PXY_CMD: commands to run
# * $CLI_OUT, $SRV_OUT, $PXY_OUT: files to contain client/server/proxy logs
#
# Outputs:
# * $CLI_EXIT: client return code
# * $SRV_RET: server return code
do_run_test_once() {
# run the commands
if [ -n "$PXY_CMD" ]; then
printf "# %s\n%s\n" "$NAME" "$PXY_CMD" > $PXY_OUT
$PXY_CMD >> $PXY_OUT 2>&1 &
PXY_PID=$!
wait_proxy_start "$PXY_PORT" "$PXY_PID"
fi
check_osrv_dtls
printf '# %s\n%s\n' "$NAME" "$SRV_CMD" > $SRV_OUT
provide_input | $SRV_CMD >> $SRV_OUT 2>&1 &
SRV_PID=$!
wait_server_start "$SRV_PORT" "$SRV_PID"
printf '# %s\n%s\n' "$NAME" "$CLI_CMD" > $CLI_OUT
eval "$CLI_CMD" >> $CLI_OUT 2>&1 &
wait_client_done
sleep 0.05
# terminate the server (and the proxy)
kill $SRV_PID
wait $SRV_PID
SRV_RET=$?
if [ -n "$PXY_CMD" ]; then
kill $PXY_PID >/dev/null 2>&1
wait $PXY_PID
fi
}
# Usage: run_test name [-p proxy_cmd] srv_cmd cli_cmd cli_exit [option [...]]
# Options: -s pattern pattern that must be present in server output
# -c pattern pattern that must be present in client output
# -u pattern lines after pattern must be unique in client output
# -f call shell function on client output
# -S pattern pattern that must be absent in server output
# -C pattern pattern that must be absent in client output
# -U pattern lines after pattern must be unique in server output
# -F call shell function on server output
# -g call shell function on server and client output
run_test() {
NAME="$1"
shift 1
if is_excluded "$NAME"; then
SKIP_NEXT="NO"
# There was no request to run the test, so don't record its outcome.
return
fi
print_name "$NAME"
# Do we only run numbered tests?
if [ -n "$RUN_TEST_NUMBER" ]; then
case ",$RUN_TEST_NUMBER," in
*",$TESTS,"*) :;;
*) SKIP_NEXT="YES";;
esac
fi
# does this test use a proxy?
if [ "X$1" = "X-p" ]; then
PXY_CMD="$2"
shift 2
else
PXY_CMD=""
fi
# get commands and client output
SRV_CMD="$1"
CLI_CMD="$2"
CLI_EXPECT="$3"
shift 3
# Check if test uses files
case "$SRV_CMD $CLI_CMD" in
*data_files/*)
requires_config_enabled MBEDTLS_FS_IO;;
esac
# If the client or serve requires a ciphersuite, check that it's enabled.
maybe_requires_ciphersuite_enabled "$SRV_CMD" "$@"
maybe_requires_ciphersuite_enabled "$CLI_CMD" "$@"
# should we skip?
if [ "X$SKIP_NEXT" = "XYES" ]; then
SKIP_NEXT="NO"
record_outcome "SKIP"
SKIPS=$(( $SKIPS + 1 ))
return
fi
analyze_test_commands "$@"
TIMES_LEFT=2
while [ $TIMES_LEFT -gt 0 ]; do
TIMES_LEFT=$(( $TIMES_LEFT - 1 ))
do_run_test_once
check_test_failure "$@"
case $outcome in
PASS) break;;
RETRY*) printf "$outcome ";;
FAIL) return;;
esac
done
# If we get this far, the test case passed.
record_outcome "PASS"
if [ "$PRESERVE_LOGS" -gt 0 ]; then
mv $SRV_OUT o-srv-${TESTS}.log