Rework mbedtls_test_unhexify()

Rework mbedtls_test_unhexify to extend its scope of usage.
Return in error when the function detects an error instead
of calling mbedtls_exit().
Improve safety by checking the output buffer is not overrun.

Signed-off-by: Ronald Cron <ronald.cron@arm.com>
diff --git a/tests/src/helpers.c b/tests/src/helpers.c
index 08d88a5..b9abf19 100644
--- a/tests/src/helpers.c
+++ b/tests/src/helpers.c
@@ -41,38 +41,49 @@
 #endif /* MBEDTLS_PLATFORM_C */
 }
 
-int mbedtls_test_unhexify( unsigned char *obuf, const char *ibuf )
+static int ascii2uc(const char c, unsigned char *uc)
 {
-    unsigned char c, c2;
-    int len = strlen( ibuf ) / 2;
-    TEST_HELPER_ASSERT( strlen( ibuf ) % 2 == 0 ); /* must be even number of bytes */
+    if( ( c >= '0' ) && ( c <= '9' ) )
+        *uc = c - '0';
+    else if( ( c >= 'a' ) && ( c <= 'f' ) )
+        *uc = c - 'a' + 10;
+    else if( ( c >= 'A' ) && ( c <= 'F' ) )
+        *uc = c - 'A' + 10;
+    else
+        return( -1 );
+
+    return( 0 );
+}
+
+int mbedtls_test_unhexify( unsigned char *obuf,
+                           size_t obufmax,
+                           const char *ibuf,
+                           size_t *len )
+{
+    unsigned char uc, uc2;
+
+    *len = strlen( ibuf );
+
+    /* Must be even number of bytes. */
+    if ( ( *len ) & 1 )
+        return( -1 );
+    *len /= 2;
+
+    if ( (*len) > obufmax )
+        return( -1 );
 
     while( *ibuf != 0 )
     {
-        c = *ibuf++;
-        if( c >= '0' && c <= '9' )
-            c -= '0';
-        else if( c >= 'a' && c <= 'f' )
-            c -= 'a' - 10;
-        else if( c >= 'A' && c <= 'F' )
-            c -= 'A' - 10;
-        else
-            TEST_HELPER_ASSERT( 0 );
+        if ( ascii2uc( *(ibuf++), &uc ) != 0 )
+            return( -1 );
 
-        c2 = *ibuf++;
-        if( c2 >= '0' && c2 <= '9' )
-            c2 -= '0';
-        else if( c2 >= 'a' && c2 <= 'f' )
-            c2 -= 'a' - 10;
-        else if( c2 >= 'A' && c2 <= 'F' )
-            c2 -= 'A' - 10;
-        else
-            TEST_HELPER_ASSERT( 0 );
+        if ( ascii2uc( *(ibuf++), &uc2 ) != 0 )
+            return( -1 );
 
-        *obuf++ = ( c << 4 ) | c2;
+        *(obuf++) = ( uc << 4 ) | uc2;
     }
 
-    return len;
+    return( 0 );
 }
 
 void mbedtls_test_hexify( unsigned char *obuf,
@@ -117,6 +128,7 @@
 unsigned char *mbedtls_test_unhexify_alloc( const char *ibuf, size_t *olen )
 {
     unsigned char *obuf;
+    size_t len;
 
     *olen = strlen( ibuf ) / 2;
 
@@ -125,8 +137,7 @@
 
     obuf = mbedtls_calloc( 1, *olen );
     TEST_HELPER_ASSERT( obuf != NULL );
-
-    (void) mbedtls_test_unhexify( obuf, ibuf );
+    TEST_HELPER_ASSERT( mbedtls_test_unhexify( obuf, *olen, ibuf, &len ) == 0 );
 
     return( obuf );
 }