commit 22dc2e7a9b5dde165b1af00f7a118cc85c00484d
author Gilles Peskine <Gilles.Peskine@arm.com> Tue Feb 02 21:06:10 2021 +0100
committer Gilles Peskine <Gilles.Peskine@arm.com> Fri Feb 12 15:57:03 2021 +0100
tree 1f4058c1f6598e9f633cc50fb0046e710d6eb482
parent d7e82ad9bf96f2bca5f9468e6f0dd2cc60071823

Fix mutex leak in RSA

mbedtls_rsa_gen_key() was not freeing the RSA object, and specifically
not freeing the mutex, in some error cases.

Signed-off-by: Gilles Peskine <Gilles.Peskine@arm.com>

library/rsa.c

diff --git a/library/rsa.c b/library/rsa.c
index 65b75d6..0fa19e5 100644
--- a/library/rsa.c
+++ b/library/rsa.c
@@ -538,15 +538,15 @@
     int ret;
     mbedtls_mpi H, G;
 
-    if( f_rng == NULL || nbits < 128 || exponent < 3 )
-        return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
-
-    if( nbits % 2 )
-        return( MBEDTLS_ERR_RSA_BAD_INPUT_DATA );
-
     mbedtls_mpi_init( &H );
     mbedtls_mpi_init( &G );
 
+    if( f_rng == NULL || nbits < 128 || exponent < 3 || nbits % 2 != 0 )
+    {
+        ret = MBEDTLS_ERR_RSA_BAD_INPUT_DATA;
+        goto cleanup;
+    }
+
     /*
      * find primes P and Q with Q < P so that:
      * GCD( E, (P-1)*(Q-1) ) == 1
@@ -610,7 +610,9 @@
     if( ret != 0 )
     {
         mbedtls_rsa_free( ctx );
-        return( MBEDTLS_ERR_RSA_KEY_GEN_FAILED + ret );
+        if( ( -ret & ~0x7f ) == 0 )
+            ret = MBEDTLS_ERR_RSA_KEY_GEN_FAILED + ret;
+        return( ret );
     }
 
     return( 0 );