Merge pull request #3248 from gilles-peskine-arm/travis-rationalize-2.7

Backport 2.7: Rationalize Travis builds

.travis.yml

@@ -1,41 +1,64 @@
 language: c
-compiler:
-- clang
-- gcc
+compiler: gcc
 sudo: false
 cache: ccache
-script:
-- tests/scripts/recursion.pl library/*.c
-- tests/scripts/check-generated-files.sh
-- tests/scripts/check-doxy-blocks.pl
-- tests/scripts/check-names.sh
-- tests/scripts/check-files.py
-- tests/scripts/doxygen.sh
-- cmake -D CMAKE_BUILD_TYPE:String="Check" .
-- make
-- make test
-- programs/test/selftest
-- OSSL_NO_DTLS=1 tests/compat.sh
-- tests/ssl-opt.sh -e '\(DTLS\|SCSV\).*openssl'
-- tests/scripts/test-ref-configs.pl
-- tests/scripts/curves.pl
-- tests/scripts/key-exchanges.pl
+
+jobs:
+  include:
+    - name: basic checks and reference configurations
+      addons:
+        apt:
+          packages:
+          - gnutls-bin
+          - doxygen
+          - graphviz
+          - gcc-arm-none-eabi
+          - libnewlib-arm-none-eabi
+      language: python # Needed to get pip for Python 3
+      python: 3.5 # version from Ubuntu 16.04
+      install:
+        - pip install pylint==2.4.4
+      script:
+        - tests/scripts/all.sh -k 'check_*'
+        - tests/scripts/all.sh -k test_default_out_of_box
+        - tests/scripts/test-ref-configs.pl
+        - tests/scripts/all.sh -k build_arm_none_eabi_gcc_arm5vte build_arm_none_eabi_gcc_m0plus
+
+    - name: full configuration
+      script:
+        - tests/scripts/all.sh -k test_full_cmake_gcc_asan
+
+    - name: check compilation guards
+      script:
+        - tests/scripts/all.sh -k 'test_depends_*' 'build_key_exchanges'
+
+    - name: macOS
+      os: osx
+      compiler: clang
+      script:
+        - tests/scripts/all.sh -k test_default_out_of_box
+
+    - name: Windows
+      os: windows
+      script:
+        - scripts/windows_msbuild.bat v141 # Visual Studio 2017
+
 after_failure:
 - tests/scripts/travis-log-failure.sh
+
 env:
   global:
     - SEED=1
-    - secure: "barHldniAfXyoWOD/vcO+E6/Xm4fmcaUoC9BeKW+LwsHqlDMLvugaJnmLXkSpkbYhVL61Hzf3bo0KPJn88AFc5Rkf8oYHPjH4adMnVXkf3B9ghHCgznqHsAH3choo6tnPxaFgOwOYmLGb382nQxfE5lUdvnM/W/psQjWt66A1+k="
+    - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg="
 
 addons:
   apt:
     packages:
-    - doxygen
-    - graphviz
+    - gnutls-bin
   coverity_scan:
     project:
       name: "ARMmbed/mbedtls"
-    notification_email: p.j.bakker@polarssl.org
+    notification_email: support-mbedtls@arm.com
     build_command_prepend:
     build_command: make
     branch_pattern: coverity_scan

scripts/windows_msbuild.bat

@@ -0,0 +1,20 @@
+@rem Build and test Mbed TLS with Visual Studio using msbuild.
+@rem Usage: windows_msbuild [RETARGET]
+@rem   RETARGET: version of Visual Studio to emulate
+@rem             https://docs.microsoft.com/en-us/cpp/build/how-to-modify-the-target-framework-and-platform-toolset
+
+@rem These parameters are hard-coded for now.
+set "arch=x64" & @rem "x86" or "x64"
+set "cfg=Release" & @rem "Debug" or "Release"
+set "vcvarsall=C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"
+
+if not "%~1"=="" set "retarget=,PlatformToolset=%1"
+
+@rem If the %USERPROFILE%\Source directory exists, then running
+@rem vcvarsall.bat will silently change the directory to that directory.
+@rem Setting the VSCMD_START_DIR environment variable causes it to change
+@rem to that directory instead.
+set "VSCMD_START_DIR=%~dp0\..\visualc\VS2010"
+
+"%vcvarsall%" x64 && ^
+msbuild /t:Rebuild /p:Configuration=%cfg%%retarget% /m mbedTLS.sln

tests/scripts/all.sh

@@ -131,6 +131,7 @@ pre_initialize_variables () {
     : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build}
     : ${ARMC5_BIN_DIR:=/usr/bin}
     : ${ARMC6_BIN_DIR:=/usr/bin}
+    : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-}
 
     # if MAKEFLAGS is not set add the -j option to speed up invocations of make
     if [ -z "${MAKEFLAGS+set}" ]; then
@@ -192,6 +193,9 @@ General options:
   -f|--force            Force the tests to overwrite any modified files.
   -k|--keep-going       Run all tests and report errors at the end.
   -m|--memory           Additional optional memory tests.
+     --arm-none-eabi-gcc-prefix=<string>
+                        Prefix for a cross-compiler for arm-none-eabi
+                        (default: "${ARM_NONE_EABI_GCC_PREFIX}")
      --armcc            Run ARM Compiler builds (on by default).
      --except           Exclude the COMPONENTs listed on the command line,
                         instead of running only those.
@@ -277,9 +281,13 @@ armc6_build_test()
 {
     FLAGS="$1"
 
-    msg "build: ARM Compiler 6 ($FLAGS), make"
+    msg "build: ARM Compiler 6 ($FLAGS)"
     ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \
                     WARNING_CFLAGS='-xc -std=c99' make lib
+
+    msg "size: ARM Compiler 6 ($FLAGS)"
+    "$ARMC6_FROMELF" -z library/*.o
+
     make clean
 }
 
@@ -305,6 +313,7 @@ pre_parse_command_line () {
 
     while [ $# -gt 0 ]; do
         case "$1" in
+            --arm-none-eabi-gcc-prefix) shift; ARM_NONE_EABI_GCC_PREFIX="$1";;
             --armcc) no_armcc=;;
             --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";;
             --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";;
@@ -517,7 +526,7 @@ pre_check_tools () {
     esac
 
     case " $RUN_COMPONENTS " in
-        *_arm_none_eabi_gcc[_\ ]*) check_tools "arm-none-eabi-gcc";;
+        *_arm_none_eabi_gcc[_\ ]*) check_tools "${ARM_NONE_EABI_GCC_PREFIX}gcc";;
     esac
 
     case " $RUN_COMPONENTS " in
@@ -528,9 +537,12 @@ pre_check_tools () {
         *_armcc*|*_yotta*)
             ARMC5_CC="$ARMC5_BIN_DIR/armcc"
             ARMC5_AR="$ARMC5_BIN_DIR/armar"
+            ARMC5_FROMELF="$ARMC5_BIN_DIR/fromelf"
             ARMC6_CC="$ARMC6_BIN_DIR/armclang"
             ARMC6_AR="$ARMC6_BIN_DIR/armar"
-            check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC6_CC" "$ARMC6_AR";;
+            ARMC6_FROMELF="$ARMC6_BIN_DIR/fromelf"
+            check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC5_FROMELF" \
+                        "$ARMC6_CC" "$ARMC6_AR" "$ARMC6_FROMELF";;
     esac
 
     msg "info: output_env.sh"
@@ -619,7 +631,7 @@ component_test_default_out_of_box () {
     make test
 
     msg "selftest: make, default config (out-of-box)" # ~10s
-    programs/test/selftest
+    if_build_succeeded programs/test/selftest
 }
 
 component_build_yotta () {
@@ -640,6 +652,9 @@ component_test_default_cmake_gcc_asan () {
     msg "test: main suites (inc. selftests) (ASan build)" # ~ 50s
     make test
 
+    msg "test: selftest (ASan build)" # ~ 10s
+    if_build_succeeded programs/test/selftest
+
     msg "test: ssl-opt.sh (ASan build)" # ~ 1 min
     if_build_succeeded tests/ssl-opt.sh
 
@@ -656,6 +671,9 @@ component_test_full_cmake_gcc_asan () {
     msg "test: main suites (inc. selftests) (full config, ASan build)"
     make test
 
+    msg "test: selftest (ASan build)" # ~ 10s
+    if_build_succeeded programs/test/selftest
+
     msg "test: ssl-opt.sh (full config, ASan build)"
     if_build_succeeded tests/ssl-opt.sh
 
@@ -1089,25 +1107,54 @@ component_test_have_int64 () {
 }
 
 component_build_arm_none_eabi_gcc () {
-    msg "build: arm-none-eabi-gcc, make" # ~ 10s
+    msg "build: ${ARM_NONE_EABI_GCC_PREFIX}gcc -O1" # ~ 10s
     scripts/config.pl baremetal
-    make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib
+    make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-Werror -Wall -Wextra -O1' lib
+
+    msg "size: ${ARM_NONE_EABI_GCC_PREFIX}gcc -O1"
+    ${ARM_NONE_EABI_GCC_PREFIX}size library/*.o
+}
+
+component_build_arm_none_eabi_gcc_arm5vte () {
+    msg "build: ${ARM_NONE_EABI_GCC_PREFIX}gcc -march=arm5vte" # ~ 10s
+    scripts/config.pl baremetal
+    # Build for a target platform that's close to what Debian uses
+    # for its "armel" distribution (https://wiki.debian.org/ArmEabiPort).
+    # See https://github.com/ARMmbed/mbedtls/pull/2169 and comments.
+    # It would be better to build with arm-linux-gnueabi-gcc but
+    # we don't have that on our CI at this time.
+    make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" CFLAGS='-Werror -Wall -Wextra -march=armv5te -O1' LDFLAGS='-march=armv5te' SHELL='sh -x' lib
+
+    msg "size: ${ARM_NONE_EABI_GCC_PREFIX}gcc -march=armv5te -O1"
+    ${ARM_NONE_EABI_GCC_PREFIX}size library/*.o
+}
+
+component_build_arm_none_eabi_gcc_m0plus () {
+    msg "build: ${ARM_NONE_EABI_GCC_PREFIX}gcc -mthumb -mcpu=cortex-m0plus" # ~ 10s
+    scripts/config.pl baremetal
+    make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-Werror -Wall -Wextra -mthumb -mcpu=cortex-m0plus -Os' lib
+
+    msg "size: ${ARM_NONE_EABI_GCC_PREFIX}gcc -mthumb -mcpu=cortex-m0plus -Os"
+    ${ARM_NONE_EABI_GCC_PREFIX}size library/*.o
 }
 
 component_build_arm_none_eabi_gcc_no_udbl_division () {
-    msg "build: arm-none-eabi-gcc -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s
+    msg "build: ${ARM_NONE_EABI_GCC_PREFIX} -DMBEDTLS_NO_UDBL_DIVISION, make" # ~ 10s
     scripts/config.pl baremetal
     scripts/config.pl set MBEDTLS_NO_UDBL_DIVISION
-    make CC=arm-none-eabi-gcc AR=arm-none-eabi-ar LD=arm-none-eabi-ld CFLAGS='-Werror -Wall -Wextra' lib
+    make CC="${ARM_NONE_EABI_GCC_PREFIX}gcc" AR="${ARM_NONE_EABI_GCC_PREFIX}ar" LD="${ARM_NONE_EABI_GCC_PREFIX}ld" CFLAGS='-Werror -Wall -Wextra' lib
     echo "Checking that software 64-bit division is not required"
     if_build_succeeded not grep __aeabi_uldiv library/*.o
 }
 
 component_build_armcc () {
-    msg "build: ARM Compiler 5, make"
+    msg "build: ARM Compiler 5"
     scripts/config.pl baremetal
-
     make CC="$ARMC5_CC" AR="$ARMC5_AR" WARNING_CFLAGS='--strict --c99' lib
+
+    msg "size: ARM Compiler 5"
+    "$ARMC5_FROMELF" -z library/*.o
+
     make clean
 
     # ARM Compiler 6 - Target ARMv7-A

tests/scripts/check-files.py

@@ -103,7 +103,7 @@ class LineIssueTracker(FileIssueTracker):
 
 def is_windows_file(filepath):
     _root, ext = os.path.splitext(filepath)
-    return ext in ('.dsp', '.sln', '.vcxproj')
+    return ext in ('.bat', '.dsp', '.sln', '.vcxproj')
 
 
 class PermissionIssueTracker(FileIssueTracker):
@@ -223,6 +223,7 @@ class IntegrityChecker:
         self.logger = None
         self.setup_logger(log_file)
         self.extensions_to_check = (
+            ".bat",
             ".c",
             ".data",
             ".dsp",