travis: Add FIH test scripts
Add scripts that can run instruction skip FIH tests on QEMU.
Co-authored-by: Raef Coles <raef.coles@arm.com>
Change-Id: Ia6da00174115e1dabaf84fdfc0e40476dc1b7a10
Signed-off-by: Mate Toth-Pal <mate.toth-pal@arm.com>
diff --git a/ci/fih-tests_run.sh b/ci/fih-tests_run.sh
index 7c0a9b0..7728c49 100755
--- a/ci/fih-tests_run.sh
+++ b/ci/fih-tests_run.sh
@@ -16,4 +16,4 @@
set -e
-docker run mcuboot/fih-test /bin/sh -c '/root/execute_test.sh'
\ No newline at end of file
+docker run mcuboot/fih-test /bin/sh -c '/root/execute_test.sh $0 $1 $2' 2,4,6,8,10 RELEASE SIGNATURE
\ No newline at end of file
diff --git a/ci/fih_test_docker/execute_test.sh b/ci/fih_test_docker/execute_test.sh
index 8806776..3683236 100755
--- a/ci/fih_test_docker/execute_test.sh
+++ b/ci/fih_test_docker/execute_test.sh
@@ -18,21 +18,28 @@
WORKING_DIRECTORY=/root/work/tfm
MCUBOOT_PATH=$WORKING_DIRECTORY/mcuboot
-
-TFM_DIR=/root/work/tfm/trusted-firmware-m
+TFM_DIR=$WORKING_DIRECTORY/trusted-firmware-m
TFM_BUILD_DIR=$TFM_DIR/build
-MCUBOOT_AXF=install/outputs/MPS2/AN521/bl2.axf
-SIGNED_TFM_BIN=install/outputs/MPS2/AN521/tfm_s_ns_signed.bin
-QEMU_LOG_FILE=qemu.log
-QEMU_PID_FILE=qemu_pid.txt
+
+SKIP_SIZE=$1
+BUILD_TYPE=$2
+DAMAGE_TYPE=$3
+FIH_LEVEL=$4
source ~/.bashrc
+if test -z "$FIH_LEVEL"; then
+ # Use the default level
+ CMAKE_FIH_LEVEL=""
+else
+ CMAKE_FIH_LEVEL="-DMCUBOOT_FIH_PROFILE=\"$FIH_LEVEL\""
+fi
+
# build TF-M with MCUBoot
mkdir -p $TFM_BUILD_DIR
cd $TFM_DIR
cmake -B $TFM_BUILD_DIR \
- -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DTFM_TOOLCHAIN_FILE=toolchain_GNUARM.cmake \
-DTFM_PLATFORM=mps2/an521 \
-DTEST_NS=ON \
@@ -40,21 +47,21 @@
-DTFM_PSA_API=ON \
-DMCUBOOT_PATH=$MCUBOOT_PATH \
-DMCUBOOT_LOG_LEVEL=INFO \
+ $CMAKE_FIH_LEVEL \
.
cd $TFM_BUILD_DIR
make -j install
-# Run MCUBoot and TF-M in QEMU
-/usr/bin/qemu-system-arm \
- -M mps2-an521 \
- -kernel $MCUBOOT_AXF \
- -device loader,file=$SIGNED_TFM_BIN,addr=0x10080000 \
- -chardev file,id=char0,path=$QEMU_LOG_FILE \
- -serial chardev:char0 \
- -display none \
- -pidfile $QEMU_PID_FILE \
- -daemonize
+BOOTLOADER_AXF='./install/outputs/MPS2/AN521/bl2.axf'
-sleep 7
+$MCUBOOT_PATH/ci/fih_test_docker/run_fi_test.sh $BOOTLOADER_AXF $SKIP_SIZE $DAMAGE_TYPE> fih_test_output.yaml
-cat $QEMU_LOG_FILE
\ No newline at end of file
+echo ""
+echo "test finished with"
+echo " - BUILD_TYPE: $BUILD_TYPE"
+echo " - FIH_LEVEL: $FIH_LEVEL"
+echo " - SKIP_SIZE: $SKIP_SIZE"
+echo " - DAMAGE_TYPE: $DAMAGE_TYPE"
+
+# TODO: Create human readable output
+cat fih_test_output.yaml
diff --git a/ci/fih_test_docker/fi_make_manifest.sh b/ci/fih_test_docker/fi_make_manifest.sh
new file mode 100755
index 0000000..190a316
--- /dev/null
+++ b/ci/fih_test_docker/fi_make_manifest.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Arm Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OBJDUMP=arm-none-eabi-objdump
+GDB=gdb-multiarch
+
+# Check if the ELF file specified is compatible
+if test $# -eq 0 || ! file $1 | grep "ELF" | grep "ARM" | grep "32" &>/dev/null; then
+ echo "Incompatible file: $1" 1>&2
+ exit 1
+fi
+
+# Extract the full path
+AXF_PATH=$(realpath $1)
+#Dump all objects that have a name containing FIH_LABEL
+ADDRESSES=$($OBJDUMP $AXF_PATH -t | grep "FIH_LABEL")
+# strip all data except "address, label_name"
+ADDRESSES=$(echo "$ADDRESSES" | sed "s/\([[:xdigit:]]*\).*\(FIH_LABEL_FIH_CALL_[a-zA-Z]*\)_.*/0x\1, \2/g")
+# Sort by address in ascending order
+ADDRESSES=$(echo "$ADDRESSES" | sort)
+# In the case that there is a START followed by another START take the first one
+ADDRESSES=$(echo "$ADDRESSES" | sed "N;s/\(.*START.*\)\n\(.*START.*\)/\1/;P;D")
+# Same for END except take the second one
+ADDRESSES=$(echo "$ADDRESSES" | sed "N;s/\(.*END.*\)\n\(.*END.*\)/\2/;P;D")
+
+# Output in CSV format with a label
+echo "Address, Type"
+echo "$ADDRESSES"
diff --git a/ci/fih_test_docker/fi_tester_gdb.sh b/ci/fih_test_docker/fi_tester_gdb.sh
new file mode 100755
index 0000000..05f0492
--- /dev/null
+++ b/ci/fih_test_docker/fi_tester_gdb.sh
@@ -0,0 +1,199 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Arm Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function skip_instruction {
+
+ local SKIP_ADDRESS=$1
+ local SKIP_SIZE=$2
+
+ # Parse the ASM instruction from the address using gdb
+ INSTR=$($GDB $AXF_FILE --batch -ex "disassemble $SKIP_ADDRESS" | grep "^ *$SKIP_ADDRESS" | sed "s/.*:[ \t]*\(.*\)$/\1/g")
+ # Parse the C line from the address using gdb
+ LINE=$($GDB $AXF_FILE --batch -ex "info line *$SKIP_ADDRESS" | sed "s/Line \([0-9]*\).*\"\(.*\)\".*/\2:\1/g")
+
+ # Sometimes an address is in the middle of a 4 byte instruction. In that case
+ # don't run the test
+ if test "$INSTR" == ""; then
+ return
+ fi
+
+ # Print out the meta-info about the test, in YAML
+ echo "- skip_test:"
+ echo " addr: $SKIP_ADDRESS"
+ echo " asm: \"$INSTR\""
+ echo " line: \"$LINE\""
+ echo " skip: $SKIP_SIZE"
+ # echo -ne "$SKIP_ADDRESS | $INSTR...\t"
+
+ cat >commands.gdb <<EOF
+target remote localhost: 1234
+b *$SKIP_ADDRESS
+continue&
+eval "shell sleep 0.5"
+interrupt
+if \$pc == $SKIP_ADDRESS
+ echo "Stopped at breakpoint"
+else
+ echo "Failed to stop at breakpoint"
+end
+echo "PC before increase:"
+print \$pc
+set \$pc += $SKIP_SIZE
+echo "PC after increase:"
+print \$pc
+detach
+eval "shell sleep 0.5"
+EOF
+
+ echo -n '.' 1>&2
+
+ # start qemu, dump the serial output to $QEMU_LOG_FILE
+ QEMU_LOG_FILE=qemu.log
+ QEMU_PID_FILE=qemu_pid.txt
+ rm -f $QEMU_PID_FILE $QEMU_LOG_FILE
+ /usr/bin/qemu-system-arm \
+ -M mps2-an521 \
+ -s -S \
+ -kernel $IMAGE_DIR/bl2.axf \
+ -device loader,file=$IMAGE_DIR/tfm_s_ns_signed.bin,addr=0x10080000 \
+ -chardev file,id=char0,path=$QEMU_LOG_FILE \
+ -serial chardev:char0 \
+ -display none \
+ -pidfile $QEMU_PID_FILE \
+ -daemonize
+
+ # start qemu, skip the instruction, and continue execution
+ $GDB < ./commands.gdb &>gdb_out.txt
+
+ # kill qemu
+ kill -9 `cat $QEMU_PID_FILE`
+
+ # If "Secure image initializing" is seen the TFM booted, which means that a skip
+ # managed to defeat the signature check. Write out whether the image booted or
+ # not to the log in YAML
+ if cat $QEMU_LOG_FILE | grep -i "Starting bootloader" &>/dev/null; then
+ # bootloader started successfully
+ if cat gdb_out.txt | grep -i "Stopped at breakpoint" &>/dev/null; then
+ # The target was stopped at the desired address
+ if cat $QEMU_LOG_FILE | grep -i "Secure image initializing" &>/dev/null; then
+ echo " test_exec_ok: True"
+ echo " skipped: True"
+ echo " boot: True"
+
+ #print the address that was skipped, and some context to the console
+ echo "" 1>&2
+ echo "Boot success: address: $SKIP_ADDRESS skipped: $SKIP_SIZE" 1>&2
+ arm-none-eabi-objdump -d $IMAGE_DIR/bl2.axf --start-address=$SKIP_ADDRESS -S | tail -n +7 | head -n 14 1>&2
+ echo "" 1>&2
+ echo "" 1>&2
+ else
+ LAST_LINE=`tail -n 1 $QEMU_LOG_FILE | tr -dc '[:print:]'`
+ echo " test_exec_ok: True"
+ echo " skipped: True"
+ echo " boot: False"
+ echo " last_line: '$LAST_LINE' "
+ fi
+ else
+ # The target was not stopped at the desired address.
+ # The most probable reason is that the instruction for that address is
+ # on a call path that is not taken in this run (e.g. error handling)
+ if cat $QEMU_LOG_FILE | grep -i "Secure image initializing" &>/dev/null; then
+ # The image booted, although it shouldn't happen as the test is to
+ # be run with a corrupt image.
+ echo " test_exec_ok: False"
+ echo " test_exec_fail_reason: \"No instructions were skipped (e.g. branch was not executed), but booted successfully\""
+ else
+ # the execution didn't stop at the address (e.g. the instruction
+ # is on a branch that is not taken)
+ echo " test_exec_ok: True"
+ echo " skipped: False"
+ fi
+ fi
+ else
+ # failed before the first printout
+ echo " test_exec_ok: True"
+ echo " skipped: True"
+ echo " boot: False"
+ echo " last_line: 'N/A' "
+ fi
+}
+
+# Inform how the script is used
+usage() {
+ echo "$0 <image_dir> <start_addr> [<end_addr>] [(-s | --skip) <skip_len>]"
+}
+
+#defaults
+SKIP=2
+BIN_DIR=$(pwd)/install/outputs/MPS2/AN521
+AXF_FILE=$BIN_DIR/bl2.axf
+GDB=gdb-multiarch
+BOOTLOADER=true
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ -s|--skip)
+ SKIP="$2"
+ shift
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ if test -z "$IMAGE_DIR"; then
+ IMAGE_DIR=$1
+ elif test -z "$START"; then
+ START=$1
+ elif test -z "$END"; then
+ END=$1
+ else
+ usage
+ exit 1
+ fi
+ shift
+ ;;
+ esac
+done
+
+# Check that image directory, start and end address have been supplied
+if test -z "$IMAGE_DIR"; then
+ usage
+ exit 2
+fi
+
+if test -z "$START"; then
+ usage
+ exit 2
+fi
+
+if test -z "$END"; then
+ END=$START
+fi
+
+if test -z "$SKIP"; then
+ SKIP='2'
+fi
+
+# Create the start-end address range (step 2)
+ADDRS=$(printf '0x%x\n' $(seq "$START" 2 "$END"))
+
+# For each address run the skip_instruction function on it
+for ADDR in $ADDRS; do
+ skip_instruction $ADDR $SKIP
+done
diff --git a/ci/fih_test_docker/run_fi_test.sh b/ci/fih_test_docker/run_fi_test.sh
new file mode 100755
index 0000000..bcd53f6
--- /dev/null
+++ b/ci/fih_test_docker/run_fi_test.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Copyright (c) 2020 Arm Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Get the dir this is running in and the dir the script is in.
+PWD=$(pwd)
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )
+
+# PAD is the amount of extra instructions that should be tested on each side of
+# the critical region
+PAD=6
+
+MCUBOOT_AXF=$1
+SKIP_SIZES=$2
+DAMAGE_TYPE=$3
+
+# Take an image and make it unbootable. This is done by replacing one of the
+# strings in the image with a different string. This causes the signature check
+# to fail
+function damage_image
+{
+ IMAGEDIR=$(dirname $MCUBOOT_AXF)
+ local IMAGE_NAME=tfm_s_ns_signed.bin
+ local BACKUP_IMAGE_NAME=tfm_s_ns_signed.bin.orig
+ local IMAGE=$IMAGEDIR/$IMAGE_NAME
+ mv $IMAGE $IMAGEDIR/$BACKUP_IMAGE_NAME
+
+ if [ "$DAMAGE_TYPE" = "SIGNATURE" ]; then
+ DAMAGE_PARAM="--signature"
+ elif [ "$DAMAGE_TYPE" = "IMAGE_HASH" ]; then
+ DAMAGE_PARAM="--image-hash"
+ else
+ echo "Failed to damage image $IMAGE with param $DAMAGE_TYPE" 1>&2
+ exit -1
+ fi
+
+ # TODO: damage image here
+ cp $IMAGEDIR/$BACKUP_IMAGE_NAME $IMAGE
+}
+
+function run_test
+{
+ local SKIP_SIZE=$1
+
+ $DIR/fi_make_manifest.sh $MCUBOOT_AXF > $PWD/fih_manifest.csv
+
+ # Load the CSV FI manifest file, and output in START, END lines. Effectively
+ # join START and END lines together with a comma seperator.
+ REGIONS=$(sed "N;s/\(0x[[:xdigit:]]*\).*START\n\(0x[[:xdigit:]]*\).*END.*/\1,\2/g;P;D" $PWD/fih_manifest.csv)
+ # Ignore the first line, which includes the CSV header
+ REGIONS=$(echo "$REGIONS" | tail -n+2)
+
+ for REGION in $REGIONS; do
+ #Split the START,END pairs into the two variables
+ START=$(echo $REGION | cut -d"," -f 1)
+ END=$(echo $REGION | cut -d"," -f 2)
+
+ # Apply padding, converting back to hex
+ START=$(printf "0x%X" $((START - PAD)))
+ END=$(printf "0x%X" $((END + PAD)))
+
+ # Invoke the fi tester script
+ $DIR/fi_tester_gdb.sh $IMAGEDIR $START $END --skip $SKIP_SIZE
+ done
+}
+
+damage_image $MCUBOOT_AXF
+# Run the run_test function with each skip length between min and max in turn.
+
+IFS=', ' read -r -a sizes <<< "$SKIP_SIZES"
+for size in "${sizes[@]}"; do
+ echo "Run tests with skip size $size" 1>&2
+ run_test $size
+done