Imre Kis | 87cee5b | 2025-01-15 18:52:35 +0100 | [diff] [blame] | 1 | // SPDX-FileCopyrightText: Copyright 2023-2025 Arm Limited and/or its affiliates <open-source-office@arm.com> |
| 2 | // SPDX-License-Identifier: MIT OR Apache-2.0 |
| 3 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 4 | #![allow(dead_code)] |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 5 | #![cfg_attr(not(test), no_std)] |
| 6 | |
| 7 | extern crate alloc; |
| 8 | |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 9 | use core::fmt; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 10 | use core::iter::zip; |
Imre Kis | 86fd04a | 2024-11-29 16:09:59 +0100 | [diff] [blame] | 11 | use core::panic; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 12 | |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 13 | use address::{PhysicalAddress, VirtualAddress, VirtualAddressRange}; |
Imre Kis | 86fd04a | 2024-11-29 16:09:59 +0100 | [diff] [blame] | 14 | use block::{Block, BlockIterator}; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 15 | |
| 16 | use bitflags::bitflags; |
| 17 | use packed_struct::prelude::*; |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 18 | use thiserror::Error; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 19 | |
| 20 | use self::descriptor::DescriptorType; |
| 21 | |
| 22 | use self::descriptor::{Attributes, DataAccessPermissions, Descriptor, Shareability}; |
| 23 | use self::kernel_space::KernelSpace; |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 24 | use self::page_pool::{PagePool, Pages}; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 25 | use self::region::{PhysicalRegion, VirtualRegion}; |
| 26 | use self::region_pool::{Region, RegionPool, RegionPoolError}; |
| 27 | |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 28 | pub mod address; |
Imre Kis | 86fd04a | 2024-11-29 16:09:59 +0100 | [diff] [blame] | 29 | mod block; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 30 | mod descriptor; |
Imre Kis | 725ef5e | 2024-11-20 14:20:19 +0100 | [diff] [blame] | 31 | mod granule; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 32 | pub mod kernel_space; |
| 33 | pub mod page_pool; |
| 34 | mod region; |
| 35 | mod region_pool; |
| 36 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 37 | /// Translation table error type |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 38 | #[derive(Debug, Error)] |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 39 | pub enum XlatError { |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 40 | #[error("Invalid parameter: {0}")] |
| 41 | InvalidParameterError(&'static str), |
| 42 | #[error("Cannot allocate {1}: {0:?}")] |
| 43 | PageAllocationError(RegionPoolError, usize), |
| 44 | #[error("Alignment error: {0:?} {1:?} length={2:#x} granule={3:#x}")] |
| 45 | AlignmentError(PhysicalAddress, VirtualAddress, usize, usize), |
| 46 | #[error("Entry not found for {0:?}")] |
| 47 | VaNotFound(VirtualAddress), |
| 48 | #[error("Cannot allocate virtual address {0:?}")] |
| 49 | VaAllocationError(RegionPoolError), |
| 50 | #[error("Cannot release virtual address {1:?}: {0:?}")] |
| 51 | VaReleaseError(RegionPoolError, VirtualAddress), |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 52 | } |
| 53 | |
| 54 | /// Memory attributes |
| 55 | /// |
| 56 | /// MAIR_EL1 should be configured in the same way in startup.s |
Imre Kis | 1278c9f | 2025-01-15 19:48:36 +0100 | [diff] [blame] | 57 | #[allow(non_camel_case_types)] |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 58 | #[derive(PrimitiveEnum_u8, Clone, Copy, Debug, PartialEq, Eq, Default)] |
| 59 | pub enum MemoryAttributesIndex { |
| 60 | #[default] |
| 61 | Device_nGnRnE = 0x00, |
| 62 | Normal_IWBWA_OWBWA = 0x01, |
| 63 | } |
| 64 | |
| 65 | bitflags! { |
| 66 | #[derive(Debug, Clone, Copy)] |
| 67 | pub struct MemoryAccessRights : u32 { |
| 68 | const R = 0b00000001; |
| 69 | const W = 0b00000010; |
| 70 | const X = 0b00000100; |
| 71 | const NS = 0b00001000; |
| 72 | |
| 73 | const RW = Self::R.bits() | Self::W.bits(); |
| 74 | const RX = Self::R.bits() | Self::X.bits(); |
| 75 | const RWX = Self::R.bits() | Self::W.bits() | Self::X.bits(); |
| 76 | |
| 77 | const USER = 0b00010000; |
| 78 | const DEVICE = 0b00100000; |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 79 | const GLOBAL = 0b01000000; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 80 | } |
| 81 | } |
| 82 | |
| 83 | impl From<MemoryAccessRights> for Attributes { |
| 84 | fn from(access_rights: MemoryAccessRights) -> Self { |
| 85 | let data_access_permissions = match ( |
| 86 | access_rights.contains(MemoryAccessRights::USER), |
| 87 | access_rights.contains(MemoryAccessRights::W), |
| 88 | ) { |
| 89 | (false, false) => DataAccessPermissions::ReadOnly_None, |
| 90 | (false, true) => DataAccessPermissions::ReadWrite_None, |
| 91 | (true, false) => DataAccessPermissions::ReadOnly_ReadOnly, |
| 92 | (true, true) => DataAccessPermissions::ReadWrite_ReadWrite, |
| 93 | }; |
| 94 | |
| 95 | let mem_attr_index = if access_rights.contains(MemoryAccessRights::DEVICE) { |
| 96 | MemoryAttributesIndex::Device_nGnRnE |
| 97 | } else { |
| 98 | MemoryAttributesIndex::Normal_IWBWA_OWBWA |
| 99 | }; |
| 100 | |
| 101 | Attributes { |
| 102 | uxn: !access_rights.contains(MemoryAccessRights::X) |
| 103 | || !access_rights.contains(MemoryAccessRights::USER), |
| 104 | pxn: !access_rights.contains(MemoryAccessRights::X) |
| 105 | || access_rights.contains(MemoryAccessRights::USER), |
| 106 | contiguous: false, |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 107 | not_global: !access_rights.contains(MemoryAccessRights::GLOBAL), |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 108 | access_flag: true, |
| 109 | shareability: Shareability::NonShareable, |
| 110 | data_access_permissions, |
| 111 | non_secure: access_rights.contains(MemoryAccessRights::NS), |
| 112 | mem_attr_index, |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 117 | #[derive(Debug, Clone, Copy)] |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 118 | pub enum RegimeVaRange { |
| 119 | Lower, |
| 120 | Upper, |
| 121 | } |
| 122 | |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 123 | #[derive(Debug, Clone, Copy)] |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 124 | pub enum TranslationRegime { |
| 125 | EL1_0(RegimeVaRange, u8), // EL1 and EL0 stage 1, TTBRx_EL1 |
| 126 | #[cfg(target_feature = "vh")] |
| 127 | EL2_0(RegimeVaRange, u8), // EL2 and EL0 with VHE |
| 128 | EL2, // EL2 |
| 129 | EL3, // EL3, TTBR0_EL3 |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 130 | } |
| 131 | |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 132 | impl TranslationRegime { |
| 133 | fn is_upper_va_range(&self) -> bool { |
| 134 | match self { |
| 135 | TranslationRegime::EL1_0(RegimeVaRange::Upper, _) => true, |
| 136 | #[cfg(target_feature = "vh")] |
| 137 | EL2_0(RegimeVaRange::Upper, _) => true, |
| 138 | _ => false, |
| 139 | } |
| 140 | } |
| 141 | } |
| 142 | |
Imre Kis | 725ef5e | 2024-11-20 14:20:19 +0100 | [diff] [blame] | 143 | pub type TranslationGranule<const VA_BITS: usize> = granule::TranslationGranule<VA_BITS>; |
| 144 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 145 | pub struct Xlat<const VA_BITS: usize> { |
| 146 | base_table: Pages, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 147 | page_pool: PagePool, |
| 148 | regions: RegionPool<VirtualRegion>, |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 149 | regime: TranslationRegime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 150 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 151 | } |
| 152 | |
| 153 | /// Memory translation table handling |
| 154 | /// # High level interface |
| 155 | /// * allocate and map zero initialized region (with or without VA) |
| 156 | /// * allocate and map memory region and load contents (with or without VA) |
| 157 | /// * map memory region by PA (with or without VA) |
| 158 | /// * unmap memory region by PA |
| 159 | /// * query PA by VA |
| 160 | /// * set access rights of mapped memory areas |
| 161 | /// * active mapping |
| 162 | /// |
| 163 | /// # Debug features |
| 164 | /// * print translation table details |
| 165 | /// |
| 166 | /// # Region level interface |
| 167 | /// * map regions |
| 168 | /// * unmap region |
| 169 | /// * find a mapped region which contains |
| 170 | /// * find empty area for region |
| 171 | /// * set access rights for a region |
| 172 | /// * create blocks by region |
| 173 | /// |
| 174 | /// # Block level interface |
| 175 | /// * map block |
| 176 | /// * unmap block |
| 177 | /// * set access rights of block |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 178 | impl<const VA_BITS: usize> Xlat<VA_BITS> { |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 179 | pub fn new( |
| 180 | page_pool: PagePool, |
| 181 | address: VirtualAddressRange, |
| 182 | regime: TranslationRegime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 183 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 184 | ) -> Self { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 185 | let initial_lookup_level = granule.initial_lookup_level(); |
| 186 | |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 187 | if !address.start.is_valid_in_regime::<VA_BITS>(regime) |
| 188 | || !address.end.is_valid_in_regime::<VA_BITS>(regime) |
| 189 | { |
| 190 | panic!( |
| 191 | "Invalid address range {:?} for regime {:?}", |
| 192 | address, regime |
| 193 | ); |
| 194 | } |
| 195 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 196 | let base_table = page_pool |
| 197 | .allocate_pages( |
| 198 | granule.table_size::<Descriptor>(initial_lookup_level), |
| 199 | Some(granule.table_alignment::<Descriptor>(initial_lookup_level)), |
| 200 | ) |
| 201 | .unwrap(); |
| 202 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 203 | let mut regions = RegionPool::new(); |
| 204 | regions |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 205 | .add(VirtualRegion::new(address.start, address.len().unwrap())) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 206 | .unwrap(); |
| 207 | Self { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 208 | base_table, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 209 | page_pool, |
| 210 | regions, |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 211 | regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 212 | granule, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 213 | } |
| 214 | } |
| 215 | |
| 216 | /// Allocate memory pages from the page pool, maps it to the given VA and fills it with the |
| 217 | /// initial data |
| 218 | /// # Arguments |
| 219 | /// * va: Virtual address of the memory area |
| 220 | /// * data: Data to be loaded to the memory area |
| 221 | /// * access_rights: Memory access rights of the area |
| 222 | /// # Return value |
| 223 | /// * Virtual address of the mapped memory |
| 224 | pub fn allocate_initalized_range( |
| 225 | &mut self, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 226 | va: Option<VirtualAddress>, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 227 | data: &[u8], |
| 228 | access_rights: MemoryAccessRights, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 229 | ) -> Result<VirtualAddress, XlatError> { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 230 | let mut pages = self |
| 231 | .page_pool |
| 232 | .allocate_pages(data.len(), Some(self.granule as usize)) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 233 | .map_err(|e| XlatError::PageAllocationError(e, data.len()))?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 234 | |
| 235 | pages.copy_data_to_page(data); |
| 236 | |
| 237 | let pages_length = pages.length(); |
| 238 | let physical_region = PhysicalRegion::Allocated(self.page_pool.clone(), pages); |
| 239 | let region = if let Some(required_va) = va { |
| 240 | self.regions |
| 241 | .acquire(required_va, pages_length, physical_region) |
| 242 | } else { |
Imre Kis | f0370e8 | 2024-11-18 16:24:55 +0100 | [diff] [blame] | 243 | self.regions.allocate(pages_length, physical_region, None) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 244 | } |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 245 | .map_err(XlatError::VaAllocationError)?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 246 | |
| 247 | self.map_region(region, access_rights.into()) |
| 248 | } |
| 249 | |
| 250 | /// Allocate memory pages from the page pool, maps it to the given VA and fills it with zeros |
| 251 | /// # Arguments |
| 252 | /// * va: Virtual address of the memory area |
| 253 | /// * length: Length of the memory area in bytes |
| 254 | /// * access_rights: Memory access rights of the area |
| 255 | /// # Return value |
| 256 | /// * Virtual address of the mapped memory |
| 257 | pub fn allocate_zero_init_range( |
| 258 | &mut self, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 259 | va: Option<VirtualAddress>, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 260 | length: usize, |
| 261 | access_rights: MemoryAccessRights, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 262 | ) -> Result<VirtualAddress, XlatError> { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 263 | let mut pages = self |
| 264 | .page_pool |
| 265 | .allocate_pages(length, Some(self.granule as usize)) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 266 | .map_err(|e| XlatError::PageAllocationError(e, length))?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 267 | |
| 268 | pages.zero_init(); |
| 269 | |
| 270 | let pages_length = pages.length(); |
| 271 | let physical_region = PhysicalRegion::Allocated(self.page_pool.clone(), pages); |
| 272 | let region = if let Some(required_va) = va { |
| 273 | self.regions |
| 274 | .acquire(required_va, pages_length, physical_region) |
| 275 | } else { |
Imre Kis | f0370e8 | 2024-11-18 16:24:55 +0100 | [diff] [blame] | 276 | self.regions.allocate(pages_length, physical_region, None) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 277 | } |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 278 | .map_err(XlatError::VaAllocationError)?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 279 | |
| 280 | self.map_region(region, access_rights.into()) |
| 281 | } |
| 282 | |
| 283 | /// Map memory area by physical address |
| 284 | /// # Arguments |
| 285 | /// * va: Virtual address of the memory area |
| 286 | /// * pa: Physical address of the memory area |
| 287 | /// * length: Length of the memory area in bytes |
| 288 | /// * access_rights: Memory access rights of the area |
| 289 | /// # Return value |
| 290 | /// * Virtual address of the mapped memory |
| 291 | pub fn map_physical_address_range( |
| 292 | &mut self, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 293 | va: Option<VirtualAddress>, |
| 294 | pa: PhysicalAddress, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 295 | length: usize, |
| 296 | access_rights: MemoryAccessRights, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 297 | ) -> Result<VirtualAddress, XlatError> { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 298 | let resource = PhysicalRegion::PhysicalAddress(pa); |
| 299 | let region = if let Some(required_va) = va { |
| 300 | self.regions.acquire(required_va, length, resource) |
| 301 | } else { |
Imre Kis | f0370e8 | 2024-11-18 16:24:55 +0100 | [diff] [blame] | 302 | self.regions.allocate(length, resource, None) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 303 | } |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 304 | .map_err(XlatError::VaAllocationError)?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 305 | |
| 306 | self.map_region(region, access_rights.into()) |
| 307 | } |
| 308 | |
| 309 | /// Unmap memory area by virtual address |
| 310 | /// # Arguments |
| 311 | /// * va: Virtual address |
| 312 | /// * length: Length of the memory area in bytes |
| 313 | pub fn unmap_virtual_address_range( |
| 314 | &mut self, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 315 | va: VirtualAddress, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 316 | length: usize, |
| 317 | ) -> Result<(), XlatError> { |
| 318 | let pa = self.get_pa_by_va(va, length)?; |
| 319 | |
| 320 | let region_to_release = VirtualRegion::new_with_pa(pa, va, length); |
| 321 | |
| 322 | self.unmap_region(®ion_to_release)?; |
| 323 | |
| 324 | self.regions |
| 325 | .release(region_to_release) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 326 | .map_err(|e| XlatError::VaReleaseError(e, va)) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 327 | } |
| 328 | |
| 329 | /// Query physical address by virtual address range. Only returns a value if the memory area |
| 330 | /// mapped as continuous area. |
| 331 | /// # Arguments |
| 332 | /// * va: Virtual address of the memory area |
| 333 | /// * length: Length of the memory area in bytes |
| 334 | /// # Return value |
| 335 | /// * Physical address of the mapped memory |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 336 | pub fn get_pa_by_va( |
| 337 | &self, |
| 338 | va: VirtualAddress, |
| 339 | length: usize, |
| 340 | ) -> Result<PhysicalAddress, XlatError> { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 341 | let containing_region = self |
| 342 | .find_containing_region(va, length) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 343 | .ok_or(XlatError::VaNotFound(va))?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 344 | |
| 345 | if !containing_region.used() { |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 346 | return Err(XlatError::VaNotFound(va)); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 347 | } |
| 348 | |
| 349 | Ok(containing_region.get_pa_for_va(va)) |
| 350 | } |
| 351 | |
| 352 | /// Sets the memory access right of memory area |
| 353 | /// # Arguments |
| 354 | /// * va: Virtual address of the memory area |
| 355 | /// * length: Length of the memory area in bytes |
| 356 | /// * access_rights: New memory access rights of the area |
| 357 | pub fn set_access_rights( |
| 358 | &mut self, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 359 | va: VirtualAddress, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 360 | length: usize, |
| 361 | access_rights: MemoryAccessRights, |
| 362 | ) -> Result<(), XlatError> { |
| 363 | let containing_region = self |
| 364 | .find_containing_region(va, length) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 365 | .ok_or(XlatError::VaNotFound(va))?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 366 | |
| 367 | if !containing_region.used() { |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 368 | return Err(XlatError::VaNotFound(va)); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 369 | } |
| 370 | |
| 371 | let region = VirtualRegion::new_with_pa(containing_region.get_pa_for_va(va), va, length); |
| 372 | self.map_region(region, access_rights.into())?; |
| 373 | |
| 374 | Ok(()) |
| 375 | } |
| 376 | |
| 377 | /// Activate memory mapping represented by the object |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 378 | /// |
| 379 | /// # Safety |
| 380 | /// When activating memory mapping for the running exception level, the |
| 381 | /// caller must ensure that the new mapping will not break any existing |
| 382 | /// references. After activation the caller must ensure that there are no |
| 383 | /// active references when unmapping memory. |
Imre Kis | 1278c9f | 2025-01-15 19:48:36 +0100 | [diff] [blame] | 384 | #[cfg(target_arch = "aarch64")] |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 385 | pub unsafe fn activate(&self) { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 386 | // Select translation granule |
| 387 | let is_tg0 = match &self.regime { |
| 388 | TranslationRegime::EL1_0(RegimeVaRange::Lower, _) |
| 389 | | TranslationRegime::EL2 |
| 390 | | TranslationRegime::EL3 => true, |
| 391 | TranslationRegime::EL1_0(RegimeVaRange::Upper, _) => false, |
| 392 | #[cfg(target_feature = "vh")] |
| 393 | TranslationRegime::EL2_0(RegimeVaRange::Lower, _) => true, |
| 394 | #[cfg(target_feature = "vh")] |
| 395 | TranslationRegime::EL2_0(RegimeVaRange::Upper, _) => false, |
| 396 | }; |
| 397 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 398 | if is_tg0 { |
| 399 | self.modify_tcr(|tcr| { |
| 400 | let tg0 = match self.granule { |
| 401 | TranslationGranule::Granule4k => 0b00, |
| 402 | TranslationGranule::Granule16k => 0b10, |
| 403 | TranslationGranule::Granule64k => 0b01, |
| 404 | }; |
| 405 | |
| 406 | (tcr & !(3 << 14)) | (tg0 << 14) |
| 407 | }); |
| 408 | } else { |
| 409 | self.modify_tcr(|tcr| { |
| 410 | let tg1 = match self.granule { |
| 411 | TranslationGranule::Granule4k => 0b10, |
| 412 | TranslationGranule::Granule16k => 0b01, |
| 413 | TranslationGranule::Granule64k => 0b11, |
| 414 | }; |
| 415 | |
| 416 | (tcr & !(3 << 30)) | (tg1 << 30) |
| 417 | }); |
| 418 | } |
| 419 | |
| 420 | // Set translation table |
| 421 | let base_table_pa = KernelSpace::kernel_to_pa(self.base_table.get_pa().0 as u64); |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 422 | |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 423 | match &self.regime { |
| 424 | TranslationRegime::EL1_0(RegimeVaRange::Lower, asid) => core::arch::asm!( |
| 425 | "msr ttbr0_el1, {0} |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 426 | isb", |
Imre Kis | b5146b5 | 2024-10-31 14:03:06 +0100 | [diff] [blame] | 427 | in(reg) ((*asid as u64) << 48) | base_table_pa), |
| 428 | TranslationRegime::EL1_0(RegimeVaRange::Upper, asid) => core::arch::asm!( |
| 429 | "msr ttbr1_el1, {0} |
| 430 | isb", |
| 431 | in(reg) ((*asid as u64) << 48) | base_table_pa), |
| 432 | #[cfg(target_feature = "vh")] |
| 433 | TranslationRegime::EL2_0(RegimeVaRange::Lower, asid) => core::arch::asm!( |
| 434 | "msr ttbr0_el2, {0} |
| 435 | isb", |
| 436 | in(reg) ((*asid as u64) << 48) | base_table_pa), |
| 437 | #[cfg(target_feature = "vh")] |
| 438 | TranslationRegime::EL2_0(RegimeVaRange::Upper, asid) => core::arch::asm!( |
| 439 | "msr ttbr1_el2, {0} |
| 440 | isb", |
| 441 | in(reg) ((*asid as u64) << 48) | base_table_pa), |
| 442 | TranslationRegime::EL2 => core::arch::asm!( |
| 443 | "msr ttbr0_el2, {0} |
| 444 | isb", |
| 445 | in(reg) base_table_pa), |
| 446 | TranslationRegime::EL3 => core::arch::asm!( |
| 447 | "msr ttbr0_el3, {0} |
| 448 | isb", |
| 449 | in(reg) base_table_pa), |
Imre Kis | c1dab89 | 2024-03-26 12:03:58 +0100 | [diff] [blame] | 450 | } |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 451 | } |
| 452 | |
Imre Kis | 1278c9f | 2025-01-15 19:48:36 +0100 | [diff] [blame] | 453 | /// # Safety |
| 454 | /// Dummy functions for test builds |
| 455 | #[cfg(not(target_arch = "aarch64"))] |
| 456 | pub unsafe fn activate(&self) {} |
| 457 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 458 | /// Modifies the TCR register of the selected regime of the instance. |
| 459 | #[cfg(target_arch = "aarch64")] |
| 460 | unsafe fn modify_tcr<F>(&self, f: F) |
| 461 | where |
| 462 | F: Fn(u64) -> u64, |
| 463 | { |
| 464 | let mut tcr: u64; |
| 465 | |
| 466 | match &self.regime { |
| 467 | TranslationRegime::EL1_0(_, _) => core::arch::asm!( |
| 468 | "mrs {0}, tcr_el1 |
| 469 | isb", |
| 470 | out(reg) tcr), |
| 471 | #[cfg(target_feature = "vh")] |
| 472 | TranslationRegime::EL2_0(_, _) => core::arch::asm!( |
| 473 | "mrs {0}, tcr_el2 |
| 474 | isb", |
| 475 | out(reg) tcr), |
| 476 | TranslationRegime::EL2 => core::arch::asm!( |
| 477 | "mrs {0}, tcr_el2 |
| 478 | isb", |
| 479 | out(reg) tcr), |
| 480 | TranslationRegime::EL3 => core::arch::asm!( |
| 481 | "mrs {0}, tcr_el3 |
| 482 | isb", |
| 483 | out(reg) tcr), |
| 484 | } |
| 485 | |
| 486 | tcr = f(tcr); |
| 487 | |
| 488 | match &self.regime { |
| 489 | TranslationRegime::EL1_0(_, _) => core::arch::asm!( |
| 490 | "msr tcr_el1, {0} |
| 491 | isb", |
| 492 | in(reg) tcr), |
| 493 | #[cfg(target_feature = "vh")] |
| 494 | TranslationRegime::EL2_0(_, _) => core::arch::asm!( |
| 495 | "msr tcr_el2, {0} |
| 496 | isb", |
| 497 | in(reg) tcr), |
| 498 | TranslationRegime::EL2 => core::arch::asm!( |
| 499 | "msr tcr_el2, {0} |
| 500 | isb", |
| 501 | in(reg) tcr), |
| 502 | TranslationRegime::EL3 => core::arch::asm!( |
| 503 | "msr tcr_el3, {0} |
| 504 | isb", |
| 505 | in(reg) tcr), |
| 506 | } |
| 507 | } |
| 508 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 509 | /// Prints a single translation table to the debug console |
| 510 | /// # Arguments |
| 511 | /// * level: Level of the translation table |
| 512 | /// * va: Base virtual address of the table |
| 513 | /// * table: Table entries |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 514 | fn dump_table( |
| 515 | f: &mut fmt::Formatter<'_>, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 516 | level: isize, |
| 517 | va: usize, |
| 518 | table: &[Descriptor], |
| 519 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 520 | ) -> fmt::Result { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 521 | let level_prefix = match level { |
| 522 | 0 | 1 => "|-", |
| 523 | 2 => "| |-", |
| 524 | _ => "| | |-", |
| 525 | }; |
| 526 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 527 | for (descriptor, va) in zip(table, (va..).step_by(granule.block_size_at_level(level))) { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 528 | match descriptor.get_descriptor_type(level) { |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 529 | DescriptorType::Block => { |
| 530 | writeln!( |
| 531 | f, |
| 532 | "{} {:#010x} Block -> {:#010x}", |
| 533 | level_prefix, |
| 534 | va, |
| 535 | descriptor.get_block_output_address(granule, level).0 |
| 536 | )?; |
| 537 | } |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 538 | DescriptorType::Table => { |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 539 | let table_pa = descriptor.get_next_level_table(level); |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 540 | writeln!( |
| 541 | f, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 542 | "{} {:#010x} Table -> {:#010x}", |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 543 | level_prefix, va, table_pa.0 |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 544 | )?; |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 545 | |
| 546 | let next_level_table = |
| 547 | unsafe { Self::get_table_from_pa(table_pa, granule, level + 1) }; |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 548 | Self::dump_table(f, level + 1, va, next_level_table, granule)?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 549 | } |
| 550 | _ => {} |
| 551 | } |
| 552 | } |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 553 | |
| 554 | Ok(()) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 555 | } |
| 556 | |
| 557 | /// Adds memory region from the translation table. The function splits the region to blocks and |
| 558 | /// uses the block level functions to do the mapping. |
| 559 | /// # Arguments |
| 560 | /// * region: Memory region object |
| 561 | /// # Return value |
| 562 | /// * Virtual address of the mapped memory |
| 563 | fn map_region( |
| 564 | &mut self, |
| 565 | region: VirtualRegion, |
| 566 | attributes: Attributes, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 567 | ) -> Result<VirtualAddress, XlatError> { |
Imre Kis | 86fd04a | 2024-11-29 16:09:59 +0100 | [diff] [blame] | 568 | let blocks = BlockIterator::new( |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 569 | region.get_pa(), |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 570 | region.base().remove_upper_bits::<VA_BITS>(), |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 571 | region.length(), |
| 572 | self.granule, |
| 573 | )?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 574 | for block in blocks { |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 575 | self.map_block(block, attributes.clone())?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 576 | } |
| 577 | |
| 578 | Ok(region.base()) |
| 579 | } |
| 580 | |
| 581 | /// Remove memory region from the translation table. The function splits the region to blocks |
| 582 | /// and uses the block level functions to do the unmapping. |
| 583 | /// # Arguments |
| 584 | /// * region: Memory region object |
| 585 | fn unmap_region(&mut self, region: &VirtualRegion) -> Result<(), XlatError> { |
Imre Kis | 86fd04a | 2024-11-29 16:09:59 +0100 | [diff] [blame] | 586 | let blocks = BlockIterator::new( |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 587 | region.get_pa(), |
Imre Kis | c9a55ff | 2025-01-17 15:06:50 +0100 | [diff] [blame^] | 588 | region.base().remove_upper_bits::<VA_BITS>(), |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 589 | region.length(), |
| 590 | self.granule, |
| 591 | )?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 592 | for block in blocks { |
| 593 | self.unmap_block(block); |
| 594 | } |
| 595 | |
| 596 | Ok(()) |
| 597 | } |
| 598 | |
| 599 | /// Find mapped region that contains the whole region |
| 600 | /// # Arguments |
| 601 | /// * region: Virtual address to look for |
| 602 | /// # Return value |
| 603 | /// * Reference to virtual region if found |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 604 | fn find_containing_region(&self, va: VirtualAddress, length: usize) -> Option<&VirtualRegion> { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 605 | self.regions.find_containing_region(va, length).ok() |
| 606 | } |
| 607 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 608 | /// Add block to memory mapping |
| 609 | /// # Arguments |
| 610 | /// * block: Memory block that can be represented by a single translation table entry |
| 611 | /// * attributes: Memory block's permissions, flags |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 612 | fn map_block(&mut self, block: Block, attributes: Attributes) -> Result<(), XlatError> { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 613 | Self::set_block_descriptor_recursively( |
| 614 | attributes, |
| 615 | block.pa, |
| 616 | block.va, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 617 | block.size, |
| 618 | self.granule.initial_lookup_level(), |
| 619 | unsafe { self.base_table.get_as_mut_slice::<Descriptor>() }, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 620 | &self.page_pool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 621 | &self.regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 622 | self.granule, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 623 | ) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 624 | } |
| 625 | |
| 626 | /// Adds the block descriptor to the translation table along all the intermediate tables the |
| 627 | /// reach the required granule. |
| 628 | /// # Arguments |
| 629 | /// * attributes: Memory block's permssions, flags |
| 630 | /// * pa: Physical address |
| 631 | /// * va: Virtual address |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 632 | /// * block_size: The block size in bytes |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 633 | /// * level: Translation table level |
| 634 | /// * table: Translation table on the given level |
| 635 | /// * page_pool: Page pool where the function can allocate pages for the translation tables |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 636 | /// * regime: Translation regime |
| 637 | /// * granule: Translation granule |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 638 | #[allow(clippy::too_many_arguments)] |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 639 | fn set_block_descriptor_recursively( |
| 640 | attributes: Attributes, |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 641 | pa: PhysicalAddress, |
| 642 | va: VirtualAddress, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 643 | block_size: usize, |
| 644 | level: isize, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 645 | table: &mut [Descriptor], |
| 646 | page_pool: &PagePool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 647 | regime: &TranslationRegime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 648 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 649 | ) -> Result<(), XlatError> { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 650 | // Get descriptor of the current level |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 651 | let descriptor = &mut table[va.get_level_index(granule, level)]; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 652 | |
| 653 | // We reached the required granule level |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 654 | if granule.block_size_at_level(level) == block_size { |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 655 | // Follow break-before-make sequence |
| 656 | descriptor.set_block_or_invalid_descriptor_to_invalid(level); |
| 657 | Self::invalidate(regime, Some(va)); |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 658 | descriptor.set_block_descriptor(granule, level, pa, attributes); |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 659 | return Ok(()); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 660 | } |
| 661 | |
| 662 | // Need to iterate forward |
| 663 | match descriptor.get_descriptor_type(level) { |
| 664 | DescriptorType::Invalid => { |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 665 | // Allocate page for next level table |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 666 | let mut page = page_pool |
| 667 | .allocate_pages( |
| 668 | granule.table_size::<Descriptor>(level + 1), |
| 669 | Some(granule.table_alignment::<Descriptor>(level + 1)), |
| 670 | ) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 671 | .map_err(|e| { |
| 672 | XlatError::PageAllocationError( |
| 673 | e, |
| 674 | granule.table_size::<Descriptor>(level + 1), |
| 675 | ) |
| 676 | })?; |
| 677 | |
| 678 | let next_table = unsafe { page.get_as_mut_slice() }; |
| 679 | |
| 680 | // Fill next level table |
| 681 | let result = Self::set_block_descriptor_recursively( |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 682 | attributes, |
| 683 | pa, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 684 | va.mask_for_level(granule, level), |
| 685 | block_size, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 686 | level + 1, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 687 | next_table, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 688 | page_pool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 689 | regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 690 | granule, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 691 | ); |
| 692 | |
| 693 | if result.is_ok() { |
| 694 | // Set table descriptor if the table is configured properly |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 695 | let next_table_pa = PhysicalAddress(KernelSpace::kernel_to_pa( |
| 696 | next_table.as_ptr() as u64, |
| 697 | ) as usize); |
| 698 | descriptor.set_table_descriptor(level, next_table_pa, None); |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 699 | } else { |
| 700 | // Release next level table on error and keep invalid descriptor on current level |
| 701 | page_pool.release_pages(page).unwrap(); |
| 702 | } |
| 703 | |
| 704 | result |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 705 | } |
| 706 | DescriptorType::Block => { |
| 707 | // Saving current descriptor details |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 708 | let current_va = va.mask_for_level(granule, level); |
| 709 | let current_pa = descriptor.get_block_output_address(granule, level); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 710 | let current_attributes = descriptor.get_block_attributes(level); |
| 711 | |
| 712 | // Replace block descriptor by table descriptor |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 713 | |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 714 | // Allocate page for next level table |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 715 | let mut page = page_pool |
| 716 | .allocate_pages( |
| 717 | granule.table_size::<Descriptor>(level + 1), |
| 718 | Some(granule.table_alignment::<Descriptor>(level + 1)), |
| 719 | ) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 720 | .map_err(|e| { |
| 721 | XlatError::PageAllocationError( |
| 722 | e, |
| 723 | granule.table_size::<Descriptor>(level + 1), |
| 724 | ) |
| 725 | })?; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 726 | |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 727 | let next_table = unsafe { page.get_as_mut_slice() }; |
| 728 | |
| 729 | // Explode existing block descriptor into table entries |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 730 | for exploded_va in VirtualAddressRange::new( |
| 731 | current_va, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 732 | current_va |
| 733 | .add_offset(granule.block_size_at_level(level)) |
| 734 | .unwrap(), |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 735 | ) |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 736 | .step_by(granule.block_size_at_level(level + 1)) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 737 | { |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 738 | let offset = exploded_va.diff(current_va).unwrap(); |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 739 | |
| 740 | // This call sets a single block descriptor and it should not fail |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 741 | Self::set_block_descriptor_recursively( |
| 742 | current_attributes.clone(), |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 743 | current_pa.add_offset(offset).unwrap(), |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 744 | exploded_va.mask_for_level(granule, level), |
| 745 | granule.block_size_at_level(level + 1), |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 746 | level + 1, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 747 | next_table, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 748 | page_pool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 749 | regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 750 | granule, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 751 | ) |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 752 | .unwrap(); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 753 | } |
| 754 | |
| 755 | // Invoke self to continue recursion on the newly created level |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 756 | let result = Self::set_block_descriptor_recursively( |
| 757 | attributes, |
| 758 | pa, |
| 759 | va.mask_for_level(granule, level + 1), |
| 760 | block_size, |
| 761 | level + 1, |
| 762 | next_table, |
| 763 | page_pool, |
| 764 | regime, |
| 765 | granule, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 766 | ); |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 767 | |
| 768 | if result.is_ok() { |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 769 | let next_table_pa = PhysicalAddress(KernelSpace::kernel_to_pa( |
| 770 | next_table.as_ptr() as u64, |
| 771 | ) as usize); |
| 772 | |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 773 | // Follow break-before-make sequence |
| 774 | descriptor.set_block_or_invalid_descriptor_to_invalid(level); |
| 775 | Self::invalidate(regime, Some(current_va)); |
| 776 | |
| 777 | // Set table descriptor if the table is configured properly |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 778 | descriptor.set_table_descriptor(level, next_table_pa, None); |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 779 | } else { |
| 780 | // Release next level table on error and keep invalid descriptor on current level |
| 781 | page_pool.release_pages(page).unwrap(); |
| 782 | } |
| 783 | |
| 784 | result |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 785 | } |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 786 | DescriptorType::Table => { |
| 787 | let next_level_table = unsafe { |
| 788 | Self::get_table_from_pa_mut( |
| 789 | descriptor.get_next_level_table(level), |
| 790 | granule, |
| 791 | level + 1, |
| 792 | ) |
| 793 | }; |
| 794 | |
| 795 | Self::set_block_descriptor_recursively( |
| 796 | attributes, |
| 797 | pa, |
| 798 | va.mask_for_level(granule, level), |
| 799 | block_size, |
| 800 | level + 1, |
| 801 | next_level_table, |
| 802 | page_pool, |
| 803 | regime, |
| 804 | granule, |
| 805 | ) |
| 806 | } |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 807 | } |
| 808 | } |
| 809 | |
| 810 | /// Remove block from memory mapping |
| 811 | /// # Arguments |
| 812 | /// * block: memory block that can be represented by a single translation entry |
| 813 | fn unmap_block(&mut self, block: Block) { |
| 814 | Self::remove_block_descriptor_recursively( |
| 815 | block.va, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 816 | block.size, |
| 817 | self.granule.initial_lookup_level(), |
| 818 | unsafe { self.base_table.get_as_mut_slice::<Descriptor>() }, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 819 | &self.page_pool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 820 | &self.regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 821 | self.granule, |
Imre Kis | d20b529 | 2024-12-04 16:05:30 +0100 | [diff] [blame] | 822 | ) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 823 | } |
| 824 | |
| 825 | /// Removes block descriptor from the translation table along all the intermediate tables which |
| 826 | /// become empty during the removal process. |
| 827 | /// # Arguments |
| 828 | /// * va: Virtual address |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 829 | /// * block_size: Translation block size in bytes |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 830 | /// * level: Translation table level |
| 831 | /// * table: Translation table on the given level |
| 832 | /// * page_pool: Page pool where the function can release the pages of empty tables |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 833 | /// * regime: Translation regime |
| 834 | /// * granule: Translation granule |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 835 | fn remove_block_descriptor_recursively( |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 836 | va: VirtualAddress, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 837 | block_size: usize, |
| 838 | level: isize, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 839 | table: &mut [Descriptor], |
| 840 | page_pool: &PagePool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 841 | regime: &TranslationRegime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 842 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 843 | ) { |
| 844 | // Get descriptor of the current level |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 845 | let descriptor = &mut table[va.get_level_index(granule, level)]; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 846 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 847 | // We reached the required level with the matching block size |
| 848 | if granule.block_size_at_level(level) == block_size { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 849 | descriptor.set_block_descriptor_to_invalid(level); |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 850 | Self::invalidate(regime, Some(va)); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 851 | return; |
| 852 | } |
| 853 | |
| 854 | // Need to iterate forward |
| 855 | match descriptor.get_descriptor_type(level) { |
| 856 | DescriptorType::Invalid => { |
| 857 | panic!("Cannot remove block from non-existing table"); |
| 858 | } |
| 859 | DescriptorType::Block => { |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 860 | panic!("Cannot remove block with different block size"); |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 861 | } |
| 862 | DescriptorType::Table => { |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 863 | let next_level_table = unsafe { |
| 864 | Self::get_table_from_pa_mut( |
| 865 | descriptor.get_next_level_table(level), |
| 866 | granule, |
| 867 | level + 1, |
| 868 | ) |
| 869 | }; |
| 870 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 871 | Self::remove_block_descriptor_recursively( |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 872 | va.mask_for_level(granule, level), |
| 873 | block_size, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 874 | level + 1, |
| 875 | next_level_table, |
| 876 | page_pool, |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 877 | regime, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 878 | granule, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 879 | ); |
| 880 | |
| 881 | if next_level_table.iter().all(|d| !d.is_valid()) { |
| 882 | // Empty table |
| 883 | let mut page = unsafe { |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 884 | let table_pa = descriptor.set_table_descriptor_to_invalid(level); |
| 885 | let next_table = Self::get_table_from_pa_mut(table_pa, granule, level + 1); |
| 886 | Pages::from_slice(next_table) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 887 | }; |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 888 | |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 889 | page.zero_init(); |
| 890 | page_pool.release_pages(page).unwrap(); |
| 891 | } |
| 892 | } |
| 893 | } |
| 894 | } |
| 895 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 896 | fn get_descriptor(&mut self, va: VirtualAddress, block_size: usize) -> &mut Descriptor { |
| 897 | Self::walk_descriptors( |
| 898 | va, |
| 899 | block_size, |
| 900 | self.granule.initial_lookup_level(), |
| 901 | unsafe { self.base_table.get_as_mut_slice::<Descriptor>() }, |
| 902 | self.granule, |
| 903 | ) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 904 | } |
| 905 | |
| 906 | fn walk_descriptors( |
Imre Kis | d5b96fd | 2024-09-11 17:04:32 +0200 | [diff] [blame] | 907 | va: VirtualAddress, |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 908 | block_size: usize, |
| 909 | level: isize, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 910 | table: &mut [Descriptor], |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 911 | granule: TranslationGranule<VA_BITS>, |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 912 | ) -> &mut Descriptor { |
| 913 | // Get descriptor of the current level |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 914 | let descriptor = &mut table[va.get_level_index(granule, level)]; |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 915 | |
Imre Kis | 631127d | 2024-11-21 13:09:01 +0100 | [diff] [blame] | 916 | if granule.block_size_at_level(level) == block_size { |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 917 | return descriptor; |
| 918 | } |
| 919 | |
| 920 | // Need to iterate forward |
| 921 | match descriptor.get_descriptor_type(level) { |
| 922 | DescriptorType::Invalid => { |
| 923 | panic!("Invalid descriptor"); |
| 924 | } |
| 925 | DescriptorType::Block => { |
| 926 | panic!("Cannot split existing block descriptor to table"); |
| 927 | } |
Imre Kis | a7ef684 | 2025-01-17 13:12:52 +0100 | [diff] [blame] | 928 | DescriptorType::Table => { |
| 929 | let next_level_table = unsafe { |
| 930 | Self::get_table_from_pa_mut( |
| 931 | descriptor.get_next_level_table(level), |
| 932 | granule, |
| 933 | level + 1, |
| 934 | ) |
| 935 | }; |
| 936 | |
| 937 | Self::walk_descriptors( |
| 938 | va.mask_for_level(granule, level), |
| 939 | block_size, |
| 940 | level + 1, |
| 941 | next_level_table, |
| 942 | granule, |
| 943 | ) |
| 944 | } |
| 945 | } |
| 946 | } |
| 947 | |
| 948 | /// Create a translation table descriptor slice from a physical address. |
| 949 | /// |
| 950 | /// # Safety |
| 951 | /// The caller must ensure that the physical address points to a valid translation table and |
| 952 | /// it it mapped into the virtual address space of the running kernel context. |
| 953 | unsafe fn get_table_from_pa<'a>( |
| 954 | pa: PhysicalAddress, |
| 955 | granule: TranslationGranule<VA_BITS>, |
| 956 | level: isize, |
| 957 | ) -> &'a [Descriptor] { |
| 958 | let table_va = KernelSpace::pa_to_kernel(pa.0 as u64); |
| 959 | unsafe { |
| 960 | core::slice::from_raw_parts( |
| 961 | table_va as *const Descriptor, |
| 962 | granule.entry_count_at_level(level), |
| 963 | ) |
| 964 | } |
| 965 | } |
| 966 | |
| 967 | /// Create a mutable translation table descriptor slice from a physical address. |
| 968 | /// |
| 969 | /// # Safety |
| 970 | /// The caller must ensure that the physical address points to a valid translation table and |
| 971 | /// it it mapped into the virtual address space of the running kernel context. |
| 972 | unsafe fn get_table_from_pa_mut<'a>( |
| 973 | pa: PhysicalAddress, |
| 974 | granule: TranslationGranule<VA_BITS>, |
| 975 | level: isize, |
| 976 | ) -> &'a mut [Descriptor] { |
| 977 | let table_va = KernelSpace::pa_to_kernel(pa.0 as u64); |
| 978 | unsafe { |
| 979 | core::slice::from_raw_parts_mut( |
| 980 | table_va as *mut Descriptor, |
| 981 | granule.entry_count_at_level(level), |
| 982 | ) |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 983 | } |
| 984 | } |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 985 | |
Imre Kis | 1278c9f | 2025-01-15 19:48:36 +0100 | [diff] [blame] | 986 | #[cfg(target_arch = "aarch64")] |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 987 | fn invalidate(regime: &TranslationRegime, va: Option<VirtualAddress>) { |
| 988 | // SAFETY: The assembly code invalidates the translation table entry of |
| 989 | // the VA or all entries of the translation regime. |
Imre Kis | 9a9d049 | 2024-10-31 15:19:46 +0100 | [diff] [blame] | 990 | unsafe { |
| 991 | if let Some(VirtualAddress(va)) = va { |
| 992 | match regime { |
| 993 | TranslationRegime::EL1_0(_, _) => { |
| 994 | core::arch::asm!( |
| 995 | "tlbi vaae1is, {0} |
| 996 | dsb nsh |
| 997 | isb", |
| 998 | in(reg) va) |
| 999 | } |
| 1000 | #[cfg(target_feature = "vh")] |
| 1001 | TranslationRegime::EL2_0(_, _) => { |
| 1002 | core::arch::asm!( |
| 1003 | "tlbi vaae1is, {0} |
| 1004 | dsb nsh |
| 1005 | isb", |
| 1006 | in(reg) va) |
| 1007 | } |
| 1008 | TranslationRegime::EL2 => core::arch::asm!( |
| 1009 | "tlbi vae2is, {0} |
| 1010 | dsb nsh |
| 1011 | isb", |
| 1012 | in(reg) va), |
| 1013 | TranslationRegime::EL3 => core::arch::asm!( |
| 1014 | "tlbi vae3is, {0} |
| 1015 | dsb nsh |
| 1016 | isb", |
| 1017 | in(reg) va), |
| 1018 | } |
| 1019 | } else { |
| 1020 | match regime { |
| 1021 | TranslationRegime::EL1_0(_, asid) => core::arch::asm!( |
| 1022 | "tlbi aside1, {0} |
| 1023 | dsb nsh |
| 1024 | isb", |
| 1025 | in(reg) (*asid as u64) << 48 |
| 1026 | ), |
| 1027 | #[cfg(target_feature = "vh")] |
| 1028 | TranslationRegime::EL2_0(_, asid) => core::arch::asm!( |
| 1029 | "tlbi aside1, {0} |
| 1030 | dsb nsh |
| 1031 | isb", |
| 1032 | in(reg) (*asid as u64) << 48 |
| 1033 | ), |
| 1034 | TranslationRegime::EL2 => core::arch::asm!( |
| 1035 | "tlbi alle2 |
| 1036 | dsb nsh |
| 1037 | isb" |
| 1038 | ), |
| 1039 | TranslationRegime::EL3 => core::arch::asm!( |
| 1040 | "tlbi alle3 |
| 1041 | dsb nsh |
| 1042 | isb" |
| 1043 | ), |
| 1044 | } |
| 1045 | } |
| 1046 | } |
| 1047 | } |
Imre Kis | 1278c9f | 2025-01-15 19:48:36 +0100 | [diff] [blame] | 1048 | |
| 1049 | #[cfg(not(target_arch = "aarch64"))] |
| 1050 | fn invalidate(_regime: &TranslationRegime, _va: Option<VirtualAddress>) {} |
Imre Kis | 703482d | 2023-11-30 15:51:26 +0100 | [diff] [blame] | 1051 | } |
Imre Kis | 5f96044 | 2024-11-29 16:49:43 +0100 | [diff] [blame] | 1052 | |
| 1053 | impl<const VA_BITS: usize> fmt::Debug for Xlat<VA_BITS> { |
| 1054 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> core::fmt::Result { |
| 1055 | f.debug_struct("Xlat") |
| 1056 | .field("regime", &self.regime) |
| 1057 | .field("granule", &self.granule) |
| 1058 | .field("VA_BITS", &VA_BITS) |
| 1059 | .field("base_table", &self.base_table.get_pa()) |
| 1060 | .finish()?; |
| 1061 | |
| 1062 | Self::dump_table( |
| 1063 | f, |
| 1064 | self.granule.initial_lookup_level(), |
| 1065 | 0, |
| 1066 | unsafe { self.base_table.get_as_slice() }, |
| 1067 | self.granule, |
| 1068 | )?; |
| 1069 | |
| 1070 | Ok(()) |
| 1071 | } |
| 1072 | } |