blob: 67812841f7b9aade0919a879f72bef564828fcbd [file] [log] [blame]
SimonB21ab9d72016-03-12 20:37:32 +00001#!/bin/sh
2
3# basic-build-tests.sh
4#
Bence Szépkútia2947ac2020-08-19 16:37:36 +02005# Copyright The Mbed TLS Contributors
Bence Szépkútif744bd72020-06-05 13:02:18 +02006# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
7#
8# This file is provided under the Apache License 2.0, or the
9# GNU General Public License v2.0 or later.
10#
11# **********
12# Apache License 2.0:
Bence Szépkúti51b41d52020-05-26 01:54:15 +020013#
14# Licensed under the Apache License, Version 2.0 (the "License"); you may
15# not use this file except in compliance with the License.
16# You may obtain a copy of the License at
17#
18# http://www.apache.org/licenses/LICENSE-2.0
19#
20# Unless required by applicable law or agreed to in writing, software
21# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
22# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23# See the License for the specific language governing permissions and
24# limitations under the License.
25#
Bence Szépkútif744bd72020-06-05 13:02:18 +020026# **********
27#
28# **********
29# GNU General Public License v2.0 or later:
30#
31# This program is free software; you can redistribute it and/or modify
32# it under the terms of the GNU General Public License as published by
33# the Free Software Foundation; either version 2 of the License, or
34# (at your option) any later version.
35#
36# This program is distributed in the hope that it will be useful,
37# but WITHOUT ANY WARRANTY; without even the implied warranty of
38# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
39# GNU General Public License for more details.
40#
41# You should have received a copy of the GNU General Public License along
42# with this program; if not, write to the Free Software Foundation, Inc.,
43# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
44#
45# **********
46#
SimonB21ab9d72016-03-12 20:37:32 +000047# Purpose
48#
49# Executes the basic test suites, captures the results, and generates a simple
50# test report and code coverage report.
51#
52# The tests include:
SimonB21ab9d72016-03-12 20:37:32 +000053# * Unit tests - executed using tests/scripts/run-test-suite.pl
Paul Bakker4b8bc522016-07-20 09:52:01 +010054# * Self-tests - executed using the test suites above
SimonB21ab9d72016-03-12 20:37:32 +000055# * System tests - executed using tests/ssl-opt.sh
56# * Interoperability tests - executed using tests/compat.sh
57#
58# The tests focus on functionality and do not consider performance.
59#
60# Note the tests self-adapt due to configurations in include/mbedtls/config.h
61# which can lead to some tests being skipped, and can cause the number of
Paul Bakker4b8bc522016-07-20 09:52:01 +010062# available tests to fluctuate.
SimonB21ab9d72016-03-12 20:37:32 +000063#
64# This script has been written to be generic and should work on any shell.
65#
66# Usage: basic-build-tests.sh
67#
68
69# Abort on errors (and uninitiliased variables)
70set -eu
71
72if [ -d library -a -d include -a -d tests ]; then :; else
73 echo "Must be run from mbed TLS root" >&2
74 exit 1
75fi
76
Andres AGb2fdd042016-09-22 14:17:46 +010077: ${OPENSSL:="openssl"}
78: ${OPENSSL_LEGACY:="$OPENSSL"}
79: ${GNUTLS_CLI:="gnutls-cli"}
80: ${GNUTLS_SERV:="gnutls-serv"}
81: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"}
82: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"}
83
Manuel Pégourié-Gonnardc2400d32020-06-08 12:59:27 +020084# Used to make ssl-opt.sh deterministic.
Manuel Pégourié-Gonnard1bff6842020-06-22 10:11:47 +020085#
86# See also RELEASE_SEED in all.sh. Debugging is easier if both values are kept
87# in sync. If you change the value here because it breaks some tests, you'll
88# definitely want to change it in all.sh as well.
Manuel Pégourié-Gonnardc2400d32020-06-08 12:59:27 +020089: ${SEED:=1}
90export SEED
91
Gilles Peskinebd7222d2021-07-13 23:23:23 +020092# if MAKEFLAGS is not set add the -j option to speed up invocations of make
93if [ -z "${MAKEFLAGS+set}" ]; then
94 export MAKEFLAGS="-j"
95fi
96
Andres AGb2fdd042016-09-22 14:17:46 +010097# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh
98# we just export the variables they require
99export OPENSSL_CMD="$OPENSSL"
100export GNUTLS_CLI="$GNUTLS_CLI"
101export GNUTLS_SERV="$GNUTLS_SERV"
102
Janos Follath7ccac852016-06-06 13:18:39 +0100103CONFIG_H='include/mbedtls/config.h'
104CONFIG_BAK="$CONFIG_H.bak"
SimonB21ab9d72016-03-12 20:37:32 +0000105
Janos Follathb72c6782016-07-19 14:54:17 +0100106# Step 0 - print build environment info
Andres AGb2fdd042016-09-22 14:17:46 +0100107OPENSSL="$OPENSSL" \
108 OPENSSL_LEGACY="$OPENSSL_LEGACY" \
109 GNUTLS_CLI="$GNUTLS_CLI" \
110 GNUTLS_SERV="$GNUTLS_SERV" \
111 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \
112 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \
113 scripts/output_env.sh
Janos Follathb72c6782016-07-19 14:54:17 +0100114echo
115
SimonB21ab9d72016-03-12 20:37:32 +0000116# Step 1 - Make and instrumented build for code coverage
Simon Butcherc2b0efc2016-03-18 18:28:43 +0000117export CFLAGS=' --coverage -g3 -O0 '
SimonB098a3b52016-04-16 21:56:59 +0100118make clean
Janos Follath7ccac852016-06-06 13:18:39 +0100119cp "$CONFIG_H" "$CONFIG_BAK"
SimonB098a3b52016-04-16 21:56:59 +0100120scripts/config.pl full
Gilles Peskinebd7222d2021-07-13 23:23:23 +0200121make
SimonB21ab9d72016-03-12 20:37:32 +0000122
123
124# Step 2 - Execute the tests
125TEST_OUTPUT=out_${PPID}
126cd tests
127
Gilles Peskined9701ae2020-04-09 18:33:34 +0200128if [ ! -f "seedfile" ]; then
129 dd if=/dev/urandom of="seedfile" bs=64 count=1
130fi
Gilles Peskine78c8e822020-04-09 18:50:08 +0200131echo
Gilles Peskined9701ae2020-04-09 18:33:34 +0200132
Gilles Peskine8bfe12b2020-04-09 18:29:42 +0200133# Step 2a - Unit Tests (keep going even if some tests fail)
Gilles Peskine78c8e822020-04-09 18:50:08 +0200134echo '################ Unit tests ################'
Jaeden Amero60ca6e52018-12-07 13:06:24 +0000135perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT
Gilles Peskine78c8e822020-04-09 18:50:08 +0200136echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^'
SimonB21ab9d72016-03-12 20:37:32 +0000137echo
138
Gilles Peskine8bfe12b2020-04-09 18:29:42 +0200139# Step 2b - System Tests (keep going even if some tests fail)
Gilles Peskine78c8e822020-04-09 18:50:08 +0200140echo
141echo '################ ssl-opt.sh ################'
Gilles Peskine499abc92021-07-13 23:26:00 +0200142echo "ssl-opt.sh will use SEED=$SEED for udp_proxy"
SimonB21ab9d72016-03-12 20:37:32 +0000143sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT
Gilles Peskine78c8e822020-04-09 18:50:08 +0200144echo '^^^^^^^^^^^^^^^^ ssl-opt.sh ^^^^^^^^^^^^^^^^'
SimonB21ab9d72016-03-12 20:37:32 +0000145echo
146
Gilles Peskine8bfe12b2020-04-09 18:29:42 +0200147# Step 2c - Compatibility tests (keep going even if some tests fail)
Gilles Peskine78c8e822020-04-09 18:50:08 +0200148echo '################ compat.sh ################'
149{
150 echo '#### compat.sh: Default versions'
151 sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2'
152 echo
153
154 echo '#### compat.sh: legacy (SSLv3)'
155 OPENSSL_CMD="$OPENSSL_LEGACY" sh compat.sh -m 'ssl3'
156 echo
157
158 echo '#### compat.sh: legacy (null, DES, RC4)'
159 OPENSSL_CMD="$OPENSSL_LEGACY" \
160 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \
161 sh compat.sh -e '^$' -f 'NULL\|DES\|RC4\|ARCFOUR'
162 echo
163
164 echo '#### compat.sh: next (ARIA, ChaCha)'
165 OPENSSL_CMD="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA'
166 echo
167} | tee compat-test-$TEST_OUTPUT
168echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^'
SimonB21ab9d72016-03-12 20:37:32 +0000169echo
170
171# Step 3 - Process the coverage report
172cd ..
Gilles Peskineed1f6732020-04-09 18:32:48 +0200173{
174 make lcov
175 echo SUCCESS
176} | tee tests/cov-$TEST_OUTPUT
177
178if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then
179 echo >&2 "Fatal: 'make lcov' failed"
180 exit 2
181fi
SimonB21ab9d72016-03-12 20:37:32 +0000182
183
184# Step 4 - Summarise the test report
185echo
186echo "========================================================================="
187echo "Test Report Summary"
188echo
189
Gilles Peskinea6b45822021-07-22 11:08:30 +0200190# A failure of the left-hand side of a pipe is ignored (this is a limitation
191# of sh). We'll use the presence of this file as a marker that the generation
192# of the report succeeded.
Gilles Peskine31bf22c2021-07-22 12:29:27 +0200193rm -f "tests/basic-build-test-$$.ok"
Gilles Peskinea6b45822021-07-22 11:08:30 +0200194
Gilles Peskined9437e62021-07-13 23:27:01 +0200195{
SimonB21ab9d72016-03-12 20:37:32 +0000196
Gilles Peskined9437e62021-07-13 23:27:01 +0200197 cd tests
SimonB21ab9d72016-03-12 20:37:32 +0000198
Gilles Peskined9437e62021-07-13 23:27:01 +0200199 # Step 4a - Unit tests
200 echo "Unit tests - tests/scripts/run-test-suites.pl"
SimonB21ab9d72016-03-12 20:37:32 +0000201
Gilles Peskined9437e62021-07-13 23:27:01 +0200202 PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ')
203 SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ')
204 TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ')
205 FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ')
SimonB21ab9d72016-03-12 20:37:32 +0000206
Gilles Peskined9437e62021-07-13 23:27:01 +0200207 echo "No test suites : $TOTAL_SUITES"
208 echo "Passed : $PASSED_TESTS"
209 echo "Failed : $FAILED_TESTS"
210 echo "Skipped : $SKIPPED_TESTS"
211 echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))"
212 echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))"
213 echo
SimonB21ab9d72016-03-12 20:37:32 +0000214
Gilles Peskined9437e62021-07-13 23:27:01 +0200215 TOTAL_PASS=$PASSED_TESTS
216 TOTAL_FAIL=$FAILED_TESTS
217 TOTAL_SKIP=$SKIPPED_TESTS
218 TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))
219 TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS))
SimonB21ab9d72016-03-12 20:37:32 +0000220
Gilles Peskined9437e62021-07-13 23:27:01 +0200221 # Step 4b - TLS Options tests
222 echo "TLS Options tests - tests/ssl-opt.sh"
SimonB21ab9d72016-03-12 20:37:32 +0000223
Gilles Peskined9437e62021-07-13 23:27:01 +0200224 PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p')
225 SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p')
226 TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p')
227 FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS))
SimonB21ab9d72016-03-12 20:37:32 +0000228
Gilles Peskined9437e62021-07-13 23:27:01 +0200229 echo "Passed : $PASSED_TESTS"
230 echo "Failed : $FAILED_TESTS"
231 echo "Skipped : $SKIPPED_TESTS"
232 echo "Total exec'd tests : $TOTAL_TESTS"
233 echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))"
234 echo
235
236 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS))
237 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS))
238 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS))
239 TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS))
240 TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS))
SimonB21ab9d72016-03-12 20:37:32 +0000241
242
Gilles Peskined9437e62021-07-13 23:27:01 +0200243 # Step 4c - System Compatibility tests
244 echo "System/Compatibility tests - tests/compat.sh"
SimonB21ab9d72016-03-12 20:37:32 +0000245
Gilles Peskined9437e62021-07-13 23:27:01 +0200246 PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
247 SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
248 EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
249 FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS))
SimonB21ab9d72016-03-12 20:37:32 +0000250
Gilles Peskined9437e62021-07-13 23:27:01 +0200251 echo "Passed : $PASSED_TESTS"
252 echo "Failed : $FAILED_TESTS"
253 echo "Skipped : $SKIPPED_TESTS"
254 echo "Total exec'd tests : $EXED_TESTS"
255 echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))"
256 echo
SimonB21ab9d72016-03-12 20:37:32 +0000257
Gilles Peskined9437e62021-07-13 23:27:01 +0200258 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS))
259 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS))
260 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS))
261 TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS))
262 TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS))
SimonB21ab9d72016-03-12 20:37:32 +0000263
264
Gilles Peskined9437e62021-07-13 23:27:01 +0200265 # Step 4d - Grand totals
266 echo "-------------------------------------------------------------------------"
267 echo "Total tests"
SimonB21ab9d72016-03-12 20:37:32 +0000268
Gilles Peskined9437e62021-07-13 23:27:01 +0200269 echo "Total Passed : $TOTAL_PASS"
270 echo "Total Failed : $TOTAL_FAIL"
271 echo "Total Skipped : $TOTAL_SKIP"
272 echo "Total exec'd tests : $TOTAL_EXED"
273 echo "Total avail tests : $TOTAL_AVAIL"
274 echo
SimonB21ab9d72016-03-12 20:37:32 +0000275
276
Gilles Peskined9437e62021-07-13 23:27:01 +0200277 # Step 4e - Coverage
278 echo "Coverage"
SimonB21ab9d72016-03-12 20:37:32 +0000279
Gilles Peskined9437e62021-07-13 23:27:01 +0200280 LINES_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p')
281 LINES_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p')
282 FUNCS_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p')
283 FUNCS_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) functions)$/\1/p')
284 BRANCHES_TESTED=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ branches...: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* branches)$/\1/p')
285 BRANCHES_TOTAL=$(tail -n4 cov-$TEST_OUTPUT|sed -n -e 's/ branches...: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) branches)$/\1/p')
SimonB21ab9d72016-03-12 20:37:32 +0000286
Gilles Peskined9437e62021-07-13 23:27:01 +0200287 LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL))
288 LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))"
SimonB21ab9d72016-03-12 20:37:32 +0000289
Gilles Peskined9437e62021-07-13 23:27:01 +0200290 FUNCS_PERCENT=$((1000*$FUNCS_TESTED/$FUNCS_TOTAL))
291 FUNCS_PERCENT="$(($FUNCS_PERCENT/10)).$(($FUNCS_PERCENT-($FUNCS_PERCENT/10)*10))"
SimonB21ab9d72016-03-12 20:37:32 +0000292
Gilles Peskined9437e62021-07-13 23:27:01 +0200293 BRANCHES_PERCENT=$((1000*$BRANCHES_TESTED/$BRANCHES_TOTAL))
294 BRANCHES_PERCENT="$(($BRANCHES_PERCENT/10)).$(($BRANCHES_PERCENT-($BRANCHES_PERCENT/10)*10))"
Dan Handley8cb19812020-05-28 16:20:31 +0100295
Gilles Peskined9437e62021-07-13 23:27:01 +0200296 rm unit-test-$TEST_OUTPUT
297 rm sys-test-$TEST_OUTPUT
298 rm compat-test-$TEST_OUTPUT
299 rm cov-$TEST_OUTPUT
SimonB21ab9d72016-03-12 20:37:32 +0000300
Gilles Peskined9437e62021-07-13 23:27:01 +0200301 echo "Lines Tested : $LINES_TESTED of $LINES_TOTAL $LINES_PERCENT%"
302 echo "Functions Tested : $FUNCS_TESTED of $FUNCS_TOTAL $FUNCS_PERCENT%"
303 echo "Branches Tested : $BRANCHES_TESTED of $BRANCHES_TOTAL $BRANCHES_PERCENT%"
304 echo
SimonB21ab9d72016-03-12 20:37:32 +0000305
Gilles Peskine31bf22c2021-07-22 12:29:27 +0200306 # Mark the report generation as having succeeded. This must be the
307 # last thing in the report generation.
Gilles Peskinea6b45822021-07-22 11:08:30 +0200308 touch "basic-build-test-$$.ok"
Gilles Peskined9437e62021-07-13 23:27:01 +0200309} | tee coverage-summary.txt
Janos Follath7ccac852016-06-06 13:18:39 +0100310
311make clean
312
313if [ -f "$CONFIG_BAK" ]; then
314 mv "$CONFIG_BAK" "$CONFIG_H"
315fi
Gilles Peskinef54a5de2020-04-09 18:28:14 +0200316
Gilles Peskinea6b45822021-07-22 11:08:30 +0200317# The file must exist, otherwise it means something went wrong while generating
318# the coverage report. If something did go wrong, rm will complain so this
319# script will exit with a failure status.
Gilles Peskine31bf22c2021-07-22 12:29:27 +0200320rm "tests/basic-build-test-$$.ok"