fix(expect): wait for all UARTs to pass

This change ensures that `run_package.sh`, which only runs on the
on-premises CI, sends the SIGINT signal to all non-payload UARTs once
the payload UART has completed. The Expect scripts controlling these
UARTs can trap this signal and handle it appropriately, whether that be
by ignoring it until further tests complete, or simply exiting. The busy
loop waits for all UARTs to shut down before terminating.

Previously, the busy loop would be terminated once the payload UART had
exited. This meant that in any tests where the non-payload UART had a
longer-running UART script than the payload UART, those results were not
being reported.

Signed-off-by: Chris Kay <chris.kay@arm.com>
Change-Id: I7b60c69965b250dba224cb64b1a5a10d5db1601a
diff --git a/script/run_package.sh b/script/run_package.sh
index 5db1ef1..980f19f 100755
--- a/script/run_package.sh
+++ b/script/run_package.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019-2020, Arm Limited. All rights reserved.
+# Copyright (c) 2019-2022, Arm Limited. All rights reserved.
 #
 # SPDX-License-Identifier: BSD-3-Clause
 #
@@ -405,20 +405,75 @@
 
 set +e
 pushd "$pid_dir"
+
+timeout=3600
+
+echo
+
 while :; do
-	wait -n
+	readarray -d '' all < <(find "${pid_dir}" -name 'uart*.pid' -print0)
+	readarray -d '' succeeded < <(find "${pid_dir}" -name 'uart*.success' -print0)
+	readarray -d '' failed < <(find "${pid_dir}" -name 'uart*.fail' -print0)
 
-	# Exit failure if we've any failures
-	if [ "$(wc -l < <(find -name '*.fail'))" -ne 0 ]; then
-		result=1
+	all=("${all[@]##${pid_dir}/uart}")
+	all=("${all[@]%%.pid}")
+
+	succeeded=("${succeeded[@]##${pid_dir}/uart}")
+	succeeded=("${succeeded[@]%%.success}")
+
+	failed=("${failed[@]##${pid_dir}/uart}")
+	failed=("${failed[@]%%.fail}")
+
+	completed=("${succeeded[@]}" "${failed[@]}")
+
+	readarray -t remaining < <( \
+		comm -23 \
+			<(printf '%s\n' "${all[@]}" | sort) \
+			<(printf '%s\n' "${completed[@]}" | sort) \
+	)
+
+	if [ ${#remaining[@]} = 0 ]; then
 		break
 	fi
 
-	# We're done if the payload UART exits success
-	if [ -f "$pid_dir/uart$payload_uart.success" ]; then
-		break
+	echo "Waiting ${timeout}s for ${#remaining[@]} UART(s): ${remaining[@]}"
+
+	if [[ " ${completed[@]} " =~ " ${payload_uart} " ]]; then
+		echo "- Payload (UART ${payload_uart}) completed!"
+
+		for uart in "${remaining[@]}"; do
+			pid=$(cat "${pid_dir}/uart${uart}.pid")
+
+			echo "- Terminating UART ${uart} script (PID ${pid})..."
+
+			kill -SIGINT ${pid} || true # Send Ctrl+C - don't force-kill it!
+		done
 	fi
+
+	if [ ${timeout} = 0 ]; then
+		echo "- Timeout exceeded! Killing model (PID ${model_pid})..."
+
+		kill_and_reap "${model_pid}"
+	fi
+
+	timeout=$((${timeout} - 5)) && sleep 5
 done
+
+echo
+
+if [ ${#failed[@]} != 0 ]; then
+	echo "${#failed[@]} UART(s) did not match expectations:"
+	echo
+
+	for uart in "${failed[@]}"; do
+		echo " - UART ${uart}: uart${uart}_full.txt"
+	done
+
+	echo
+
+	result=1
+fi
+
 popd
 
 # Capture whether the model is running with the 'exit model parameter' or not.