diff --git a/library/psa_crypto.c b/library/psa_crypto.c
index d8c6c1e..647ca62 100644
--- a/library/psa_crypto.c
+++ b/library/psa_crypto.c
@@ -4101,12 +4101,18 @@
                                                           slot,
                                                           alg );
 
-    if( status == PSA_SUCCESS )
-        operation->accelerator_set = 1;
-
     if( status != PSA_ERROR_NOT_SUPPORTED ||
         psa_key_lifetime_is_external( slot->attr.lifetime ) )
+    {
+        /* Indicate this operation is bound to an accelerator. When the driver
+         * setup succeeded, this indicates to the core to not call any mbedtls_
+         * functions for this operation (contexts are not interoperable).
+         * In case the drivers couldn't setup and there's no way to fallback,
+         * indicate to the core to not call mbedtls_cipher_free on an
+         * uninitialised mbed TLS cipher context. */
+        operation->accelerator_set = 1;
         goto exit;
+    }
 
     /* Proceed with initializing mbed TLS cipher context if no accelerator is
      * available for the given algorithm & key. */