Implement TLB maintenance
Invalidate TLB entries after changing descriptor to invalid and
implement break-before-make sequence when changing translation table
entries.
Signed-off-by: Imre Kis <imre.kis@arm.com>
Change-Id: I8e384de306c185315a1d189c01ff275ac646d539
diff --git a/src/descriptor.rs b/src/descriptor.rs
index bf10380..24ed8df 100644
--- a/src/descriptor.rs
+++ b/src/descriptor.rs
@@ -202,9 +202,17 @@
self.set(Self::INVALID_DESCRIPTOR_VALUE)
}
+ /// Set block or invalid descriptor to invalid
+ pub fn set_block_or_invalid_descriptor_to_invalid(&mut self, level: usize) {
+ assert!(level <= 3);
+ assert!(DescriptorType::Table != self.get_descriptor_type(level));
+
+ self.set(Self::INVALID_DESCRIPTOR_VALUE)
+ }
+
/// Set table descriptor
///
- /// **Unsafe**: The caller has to ensure that the passed next level table has the same life as
+ /// Safety: The caller has to ensure that the passed next level table has the same life as
/// the descriptor.
pub unsafe fn set_table_descriptor(
&mut self,
@@ -232,7 +240,7 @@
/// Get next level table
///
- /// **Unsafe**: The returned next level table is based on the address read from the descriptor.
+ /// Safety: The returned next level table is based on the address read from the descriptor.
/// The caller has to ensure that no other references are being used of the table.
pub unsafe fn get_next_level_table(&self, level: usize) -> &[Descriptor] {
assert!(level <= 2);
@@ -245,7 +253,7 @@
/// Get mutable next level table
///
- /// **Unsafe**: The returned next level table is based on the address read from the descriptor.
+ /// Safety: The returned next level table is based on the address read from the descriptor.
/// The caller has to ensure that no other references are being used of the table.
pub unsafe fn get_next_level_table_mut(&mut self, level: usize) -> &mut [Descriptor] {
assert!(level <= 2);
@@ -285,7 +293,13 @@
/// Set raw descriptor value
fn set(&mut self, value: u64) {
- unsafe { ptr::write_volatile(self.cell.get(), value) }
+ // Safety: The cell should point to a valid address and the assembly code is just a data
+ // barrier.
+ unsafe {
+ ptr::write_volatile(self.cell.get(), value);
+ #[cfg(target_arch = "aarch64")]
+ core::arch::asm!("dsb nsh");
+ }
}
/// Modify raw descriptor value
@@ -556,6 +570,23 @@
}
#[test]
+ fn test_descriptor_block_or_invalid_block_to_invalid() {
+ let mut descriptor = Descriptor {
+ cell: UnsafeCell::new(0),
+ };
+
+ descriptor.set_block_or_invalid_descriptor_to_invalid(0);
+ assert_eq!(0, descriptor.get());
+
+ let mut descriptor = Descriptor {
+ cell: UnsafeCell::new(3),
+ };
+
+ descriptor.set_block_or_invalid_descriptor_to_invalid(3);
+ assert_eq!(0, descriptor.get());
+ }
+
+ #[test]
#[should_panic]
fn test_descriptor_level3_to_table() {
let mut next_level_table = [Descriptor {
diff --git a/src/lib.rs b/src/lib.rs
index dd7de75..62cac5c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -560,6 +560,7 @@
1,
self.base_table.descriptors.as_mut_slice(),
&self.page_pool,
+ &self.regime,
);
}
@@ -573,6 +574,7 @@
/// * level: Translation table level
/// * table: Translation table on the given level
/// * page_pool: Page pool where the function can allocate pages for the translation tables
+ #[allow(clippy::too_many_arguments)]
fn set_block_descriptor_recursively(
attributes: Attributes,
pa: PhysicalAddress,
@@ -581,12 +583,16 @@
level: usize,
table: &mut [Descriptor],
page_pool: &PagePool,
+ regime: &TranslationRegime,
) {
// Get descriptor of the current level
let descriptor = &mut table[va.get_level_index(level)];
// We reached the required granule level
if Self::GRANULE_SIZES[level] == granule {
+ // Follow break-before-make sequence
+ descriptor.set_block_or_invalid_descriptor_to_invalid(level);
+ Self::invalidate(regime, Some(va));
descriptor.set_block_descriptor(level, pa, attributes);
return;
}
@@ -607,6 +613,7 @@
level + 1,
unsafe { descriptor.get_next_level_table_mut(level) },
page_pool,
+ regime,
)
}
DescriptorType::Block => {
@@ -638,12 +645,13 @@
level + 1,
unsafe { descriptor.get_next_level_table_mut(level) },
page_pool,
+ regime,
)
}
// Invoke self to continue recursion on the newly created level
Self::set_block_descriptor_recursively(
- attributes, pa, va, granule, level, table, page_pool,
+ attributes, pa, va, granule, level, table, page_pool, regime,
);
}
DescriptorType::Table => Self::set_block_descriptor_recursively(
@@ -654,6 +662,7 @@
level + 1,
unsafe { descriptor.get_next_level_table_mut(level) },
page_pool,
+ regime,
),
}
}
@@ -668,6 +677,7 @@
1,
self.base_table.descriptors.as_mut_slice(),
&self.page_pool,
+ &self.regime,
);
}
@@ -685,6 +695,7 @@
level: usize,
table: &mut [Descriptor],
page_pool: &PagePool,
+ regime: &TranslationRegime,
) {
// Get descriptor of the current level
let descriptor = &mut table[va.get_level_index(level)];
@@ -692,6 +703,7 @@
// We reached the required granule level
if Self::GRANULE_SIZES[level] == granule {
descriptor.set_block_descriptor_to_invalid(level);
+ Self::invalidate(regime, Some(va));
return;
}
@@ -711,6 +723,7 @@
level + 1,
next_level_table,
page_pool,
+ regime,
);
if next_level_table.iter().all(|d| !d.is_valid()) {
@@ -757,6 +770,69 @@
}
}
}
+
+ fn invalidate(regime: &TranslationRegime, va: Option<VirtualAddress>) {
+ // SAFETY: The assembly code invalidates the translation table entry of
+ // the VA or all entries of the translation regime.
+ #[cfg(target_arch = "aarch64")]
+ unsafe {
+ if let Some(VirtualAddress(va)) = va {
+ match regime {
+ TranslationRegime::EL1_0(_, _) => {
+ core::arch::asm!(
+ "tlbi vaae1is, {0}
+ dsb nsh
+ isb",
+ in(reg) va)
+ }
+ #[cfg(target_feature = "vh")]
+ TranslationRegime::EL2_0(_, _) => {
+ core::arch::asm!(
+ "tlbi vaae1is, {0}
+ dsb nsh
+ isb",
+ in(reg) va)
+ }
+ TranslationRegime::EL2 => core::arch::asm!(
+ "tlbi vae2is, {0}
+ dsb nsh
+ isb",
+ in(reg) va),
+ TranslationRegime::EL3 => core::arch::asm!(
+ "tlbi vae3is, {0}
+ dsb nsh
+ isb",
+ in(reg) va),
+ }
+ } else {
+ match regime {
+ TranslationRegime::EL1_0(_, asid) => core::arch::asm!(
+ "tlbi aside1, {0}
+ dsb nsh
+ isb",
+ in(reg) (*asid as u64) << 48
+ ),
+ #[cfg(target_feature = "vh")]
+ TranslationRegime::EL2_0(_, asid) => core::arch::asm!(
+ "tlbi aside1, {0}
+ dsb nsh
+ isb",
+ in(reg) (*asid as u64) << 48
+ ),
+ TranslationRegime::EL2 => core::arch::asm!(
+ "tlbi alle2
+ dsb nsh
+ isb"
+ ),
+ TranslationRegime::EL3 => core::arch::asm!(
+ "tlbi alle3
+ dsb nsh
+ isb"
+ ),
+ }
+ }
+ }
+ }
}
#[cfg(test)]