commit c84086e55c96a44dd63dd508a5929be1788854d1
author Yanray Wang <yanray.wang@arm.com> Fri Aug 11 15:33:07 2023 +0800
committer Yanray Wang <yanray.wang@arm.com> Fri Aug 11 16:30:37 2023 +0800
tree 722d69dbc065395ace627889174353463e8b842a
parent 45ad306fbf0b66c0396d0be442c620293c63c531

pkwrite.c: save stack usage for pk_write_key_pem

mbedtls_pk_write_key_pem would allocate 5679 bytes in writing a DER
encoded RSA private key. To save stack usage significantly, we use
heap memory instead.

Signed-off-by: Yanray Wang <yanray.wang@arm.com>

library/pkwrite.c

diff --git a/library/pkwrite.c b/library/pkwrite.c
index 511e222..f3acec7 100644
--- a/library/pkwrite.c
+++ b/library/pkwrite.c
@@ -786,7 +786,11 @@
 int mbedtls_pk_write_key_pem(const mbedtls_pk_context *key, unsigned char *buf, size_t size)
 {
     int ret = MBEDTLS_ERR_ERROR_CORRUPTION_DETECTED;
-    unsigned char output_buf[PRV_DER_MAX_BYTES];
+    unsigned char *output_buf = NULL;
+    output_buf = calloc(1, PRV_DER_MAX_BYTES);
+    if (output_buf == NULL) {
+        return MBEDTLS_ERR_PK_ALLOC_FAILED;
+    }
     const char *begin, *end;
     size_t olen = 0;
 #if defined(MBEDTLS_PK_HAVE_ECC_KEYS)
@@ -799,7 +803,8 @@
     int is_rsa_opaque = 0;
 #endif
 
-    if ((ret = mbedtls_pk_write_key_der(key, output_buf, sizeof(output_buf))) < 0) {
+    if ((ret = mbedtls_pk_write_key_der(key, output_buf, PRV_DER_MAX_BYTES)) < 0) {
+        free(output_buf);
         return ret;
     }
 
@@ -843,14 +848,19 @@
         }
     } else
 #endif /* MBEDTLS_PK_HAVE_ECC_KEYS */
-    return MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE;
+    {
+        free(output_buf);
+        return MBEDTLS_ERR_PK_FEATURE_UNAVAILABLE;
+    }
 
     if ((ret = mbedtls_pem_write_buffer(begin, end,
-                                        output_buf + sizeof(output_buf) - ret,
+                                        output_buf + PRV_DER_MAX_BYTES - ret,
                                         ret, buf, size, &olen)) != 0) {
+        free(output_buf);
         return ret;
     }
 
+    free(output_buf);
     return 0;
 }
 #endif /* MBEDTLS_PEM_WRITE_C */