diff --git a/library/psa_crypto.c b/library/psa_crypto.c
index 931e2e9..4efebbb 100644
--- a/library/psa_crypto.c
+++ b/library/psa_crypto.c
@@ -1197,7 +1197,8 @@
 static psa_status_t psa_remove_key_data_from_memory( psa_key_slot_t *slot )
 {
 #if defined(MBEDTLS_PSA_CRYPTO_SE_C)
-    if( psa_key_slot_is_external( slot ) )
+    if( psa_get_se_driver( slot->attr.lifetime, NULL, NULL ) &&
+        psa_key_slot_is_external( slot ) )
     {
         /* No key material to clean. */
     }