diff --git a/boot/bootutil/src/encrypted.c b/boot/bootutil/src/encrypted.c
index 212774e..44975cc 100644
--- a/boot/bootutil/src/encrypted.c
+++ b/boot/bootutil/src/encrypted.c
@@ -681,7 +681,7 @@
 
 void
 boot_encrypt(struct enc_key_data *enc_state, int image_index,
-        const struct flash_area *fap, uint32_t off, uint32_t sz,
+        int fa_id, uint32_t off, uint32_t sz,
         uint32_t blk_off, uint8_t *buf)
 {
     struct enc_key_data *enc;
@@ -701,7 +701,7 @@
     nonce[14] = (uint8_t)(off >> 8);
     nonce[15] = (uint8_t)off;
 
-    rc = flash_area_id_to_multi_image_slot(image_index, flash_area_get_id(fap));
+    rc = flash_area_id_to_multi_image_slot(image_index, fa_id);
     if (rc < 0) {
         assert(0);
         return;
diff --git a/boot/bootutil/src/image_validate.c b/boot/bootutil/src/image_validate.c
index a697676..b4e8b79 100644
--- a/boot/bootutil/src/image_validate.c
+++ b/boot/bootutil/src/image_validate.c
@@ -150,7 +150,7 @@
             /* Only payload is encrypted (area between header and TLVs) */
             if (off >= hdr_size && off < tlv_off) {
                 blk_off = (off - hdr_size) & 0xf;
-                boot_encrypt(enc_state, image_index, fap, off - hdr_size,
+                boot_encrypt(enc_state, image_index, flash_area_get_id(fap), off - hdr_size,
                         blk_sz, blk_off, tmp_buf);
             }
         }
diff --git a/boot/bootutil/src/loader.c b/boot/bootutil/src/loader.c
index 54e4d9b..ef1dfe8 100644
--- a/boot/bootutil/src/loader.c
+++ b/boot/bootutil/src/loader.c
@@ -1218,10 +1218,13 @@
     uint32_t off;
     uint32_t tlv_off;
     size_t blk_off;
+    int enc_area_id;
     struct image_header *hdr;
     uint16_t idx;
     uint32_t blk_sz;
     uint8_t image_index;
+    bool encrypted_src;
+    bool encrypted_dst;
 #endif
 
     TARGET_STATIC uint8_t buf[BUF_SZ] __attribute__((aligned(4)));
@@ -1245,25 +1248,22 @@
 
 #ifdef MCUBOOT_ENC_IMAGES
         image_index = BOOT_CURR_IMG(state);
-        if ((flash_area_get_id(fap_src) == FLASH_AREA_IMAGE_SECONDARY(image_index) ||
-             flash_area_get_id(fap_dst) == FLASH_AREA_IMAGE_SECONDARY(image_index)) &&
-            !(flash_area_get_id(fap_src) == FLASH_AREA_IMAGE_SECONDARY(image_index) &&
-              flash_area_get_id(fap_dst) == FLASH_AREA_IMAGE_SECONDARY(image_index))) {
-            /* assume the secondary slot as src, needs decryption */
-            hdr = boot_img_hdr(state, BOOT_SECONDARY_SLOT);
-#if !defined(MCUBOOT_SWAP_USING_MOVE)
-            off = off_src;
-            if (flash_area_get_id(fap_dst) == FLASH_AREA_IMAGE_SECONDARY(image_index)) {
-                /* might need encryption (metadata from the primary slot) */
-                hdr = boot_img_hdr(state, BOOT_PRIMARY_SLOT);
-                off = off_dst;
-            }
-#else
+        encrypted_src = (flash_area_get_id(fap_src) != FLASH_AREA_IMAGE_PRIMARY(image_index));
+        encrypted_dst = (flash_area_get_id(fap_dst) != FLASH_AREA_IMAGE_PRIMARY(image_index));
+
+        if (encrypted_src != encrypted_dst) {
             off = off_dst;
-            if (flash_area_get_id(fap_dst) == FLASH_AREA_IMAGE_SECONDARY(image_index)) {
+
+            if (encrypted_dst) {
+                /* Need encryption, metadata from the primary slot */
                 hdr = boot_img_hdr(state, BOOT_PRIMARY_SLOT);
+                enc_area_id = FLASH_AREA_IMAGE_PRIMARY(image_index);
+            } else {
+                /* Need decryption, metadata from the secondary slot */
+                hdr = boot_img_hdr(state, BOOT_SECONDARY_SLOT);
+                enc_area_id = FLASH_AREA_IMAGE_SECONDARY(image_index);
             }
-#endif
+
             if (IS_ENCRYPTED(hdr)) {
                 uint32_t abs_off = off + bytes_copied;
                 if (abs_off < hdr->ih_hdr_size) {
@@ -1294,7 +1294,7 @@
                             blk_sz = tlv_off - abs_off;
                         }
                     }
-                    boot_encrypt(BOOT_CURR_ENC(state), image_index, fap_src,
+                    boot_encrypt(BOOT_CURR_ENC(state), image_index, enc_area_id,
                             (abs_off + idx) - hdr->ih_hdr_size, blk_sz,
                             blk_off, &buf[idx]);
                 }
@@ -2774,13 +2774,13 @@
              * Part of the chunk is encrypted payload */
             blk_off = ((bytes_copied) - hdr->ih_hdr_size) & 0xf;
             blk_sz = tlv_off - (bytes_copied);
-            boot_encrypt(BOOT_CURR_ENC(state), image_index, fap_src,
+            boot_encrypt(BOOT_CURR_ENC(state), image_index, area_id,
                 (bytes_copied + idx) - hdr->ih_hdr_size, blk_sz,
                 blk_off, cur_dst);
         } else {
             /* Image encrypted payload section */
             blk_off = ((bytes_copied) - hdr->ih_hdr_size) & 0xf;
-            boot_encrypt(BOOT_CURR_ENC(state), image_index, fap_src,
+            boot_encrypt(BOOT_CURR_ENC(state), image_index, area_id,
                     (bytes_copied + idx) - hdr->ih_hdr_size, blk_sz,
                     blk_off, cur_dst);
         }