diff --git a/ostd/src/arch/x86/iommu/invalidate/descriptor/mod.rs b/ostd/src/arch/x86/iommu/invalidate/descriptor/mod.rs new file mode 100644 index 000000000..b4e656c78 --- /dev/null +++ b/ostd/src/arch/x86/iommu/invalidate/descriptor/mod.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MPL-2.0 + +pub struct InterruptEntryCache(pub u128); + +impl InterruptEntryCache { + pub fn global_invalidation() -> Self { + Self(0x4) + } +} diff --git a/ostd/src/arch/x86/iommu/invalidate/mod.rs b/ostd/src/arch/x86/iommu/invalidate/mod.rs new file mode 100644 index 000000000..e909ce3d6 --- /dev/null +++ b/ostd/src/arch/x86/iommu/invalidate/mod.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MPL-2.0 + +use log::{info, warn}; +use queue::Queue; +use spin::Once; + +use super::registers::{ExtendedCapabilityFlags, IOMMU_REGS}; +use crate::sync::SpinLock; + +pub mod descriptor; +pub mod queue; + +pub(super) fn init() { + let mut iommu_regs = IOMMU_REGS.get().unwrap().lock(); + if !iommu_regs + .extended_capability() + .flags() + .contains(ExtendedCapabilityFlags::QI) + { + warn!("[IOMMU] Queued invalidation not supported"); + return; + } + + QUEUE.call_once(|| { + let queue = Queue::new(); + iommu_regs.enable_queued_invalidation(&queue); + SpinLock::new(queue) + }); + + info!("[IOMMU] Queued invalidation is enabled"); +} + +pub(super) static QUEUE: Once> = Once::new(); diff --git a/ostd/src/arch/x86/iommu/invalidate/queue.rs b/ostd/src/arch/x86/iommu/invalidate/queue.rs new file mode 100644 index 000000000..f8da4a41c --- /dev/null +++ b/ostd/src/arch/x86/iommu/invalidate/queue.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: MPL-2.0 + +use core::mem::size_of; + +use crate::{ + mm::{FrameAllocOptions, Segment, VmIo, PAGE_SIZE}, + prelude::Paddr, +}; + +pub struct Queue { + segment: Segment, + queue_size: usize, + tail: usize, +} + +impl Queue { + pub fn append_descriptor(&mut self, descriptor: u128) { + if self.tail == self.queue_size { + self.tail = 0; + } + self.segment + .write_val(self.tail * size_of::(), &descriptor) + .unwrap(); + self.tail += 1; + } + + pub fn tail(&self) -> usize { + self.tail + } + + pub fn size(&self) -> usize { + self.queue_size + } + + pub(crate) fn base_paddr(&self) -> Paddr { + self.segment.start_paddr() + } + + pub(super) fn new() -> Self { + const DEFAULT_PAGES: usize = 1; + let segment = FrameAllocOptions::new(DEFAULT_PAGES) + .is_contiguous(true) + .alloc_contiguous() + .unwrap(); + Self { + segment, + queue_size: (DEFAULT_PAGES * PAGE_SIZE) / size_of::(), + tail: 0, + } + } +} diff --git a/ostd/src/arch/x86/iommu/mod.rs b/ostd/src/arch/x86/iommu/mod.rs index 599bf0a16..1c23f4e12 100644 --- a/ostd/src/arch/x86/iommu/mod.rs +++ b/ostd/src/arch/x86/iommu/mod.rs @@ -4,6 +4,7 @@ mod dma_remapping; mod fault; +mod invalidate; mod registers; pub(crate) use dma_remapping::{has_dma_remapping, map, unmap}; @@ -21,6 +22,7 @@ pub enum IommuError { pub(crate) fn init() -> Result<(), IommuError> { registers::init()?; + invalidate::init(); dma_remapping::init(); Ok(()) diff --git a/ostd/src/arch/x86/iommu/registers/invalidation.rs b/ostd/src/arch/x86/iommu/registers/invalidation.rs new file mode 100644 index 000000000..4df3c773b --- /dev/null +++ b/ostd/src/arch/x86/iommu/registers/invalidation.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! Invalidation-related registers + +use volatile::{ + access::{ReadOnly, ReadWrite, WriteOnly}, + Volatile, +}; + +use super::ExtendedCapability; +use crate::prelude::Vaddr; + +#[derive(Debug)] +pub struct InvalidationRegisters { + pub(super) queue_head: Volatile<&'static u64, ReadOnly>, + pub(super) queue_tail: Volatile<&'static mut u64, ReadWrite>, + pub(super) queue_addr: Volatile<&'static mut u64, ReadWrite>, + + pub(super) completion_status: Volatile<&'static mut u32, ReadWrite>, + pub(super) _completion_event_control: Volatile<&'static mut u32, ReadWrite>, + pub(super) _completion_event_data: Volatile<&'static mut u32, ReadWrite>, + pub(super) _completion_event_addr: Volatile<&'static mut u32, ReadWrite>, + pub(super) _completion_event_upper_addr: Volatile<&'static mut u32, ReadWrite>, + + pub(super) _queue_error_record: Volatile<&'static mut u64, ReadOnly>, + + pub(super) _invalidate_address: Volatile<&'static mut u64, WriteOnly>, + pub(super) _iotlb_invalidate: Volatile<&'static mut u64, ReadWrite>, +} + +impl InvalidationRegisters { + /// Create an instance from IOMMU base address. + /// + /// # Safety + /// + /// User must ensure the address is valid. + pub(super) unsafe fn new(base_vaddr: Vaddr) -> Self { + let extended_capability: Volatile<&u64, ReadOnly> = + Volatile::new_read_only(&*((base_vaddr + 0x10) as *const u64)); + let extend_cap = ExtendedCapability::new(extended_capability.read()); + let offset = extend_cap.iotlb_register_offset() as usize * 16; + + let invalidate_address = + Volatile::new_write_only(&mut *((base_vaddr + offset) as *mut u64)); + let iotlb_invalidate = Volatile::new(&mut *((base_vaddr + offset + 0x8) as *mut u64)); + + Self { + queue_head: Volatile::new_read_only(&*((base_vaddr + 0x80) as *mut u64)), + queue_tail: Volatile::new(&mut *((base_vaddr + 0x88) as *mut u64)), + queue_addr: Volatile::new(&mut *((base_vaddr + 0x90) as *mut u64)), + completion_status: Volatile::new(&mut *((base_vaddr + 0x9C) as *mut u32)), + _completion_event_control: Volatile::new(&mut *((base_vaddr + 0xA0) as *mut u32)), + _completion_event_data: Volatile::new(&mut *((base_vaddr + 0xA4) as *mut u32)), + _completion_event_addr: Volatile::new(&mut *((base_vaddr + 0xA8) as *mut u32)), + _completion_event_upper_addr: Volatile::new(&mut *((base_vaddr + 0xAC) as *mut u32)), + _queue_error_record: Volatile::new_read_only(&mut *((base_vaddr + 0xB0) as *mut u64)), + _invalidate_address: invalidate_address, + _iotlb_invalidate: iotlb_invalidate, + } + } +} diff --git a/ostd/src/arch/x86/iommu/registers/mod.rs b/ostd/src/arch/x86/iommu/registers/mod.rs index 57b1e9a46..adfc81f8e 100644 --- a/ostd/src/arch/x86/iommu/registers/mod.rs +++ b/ostd/src/arch/x86/iommu/registers/mod.rs @@ -5,12 +5,15 @@ mod capability; mod command; mod extended_cap; +mod invalidation; mod status; use bit_field::BitField; pub use capability::Capability; use command::GlobalCommand; use extended_cap::ExtendedCapability; +pub use extended_cap::ExtendedCapabilityFlags; +use invalidation::InvalidationRegisters; use log::debug; use spin::Once; use status::GlobalStatus; @@ -19,7 +22,7 @@ use volatile::{ Volatile, }; -use super::{dma_remapping::RootTable, IommuError}; +use super::{dma_remapping::RootTable, invalidate::queue::Queue, IommuError}; use crate::{ arch::{ iommu::fault, @@ -62,6 +65,8 @@ pub struct IommuRegisters { root_table_address: Volatile<&'static mut u64, ReadWrite>, #[allow(dead_code)] context_command: Volatile<&'static mut u64, ReadWrite>, + + invalidate: InvalidationRegisters, } impl IommuRegisters { @@ -106,6 +111,65 @@ impl IommuRegisters { while !self.global_status().contains(GlobalStatus::TES) {} } + pub(super) fn enable_queued_invalidation(&mut self, queue: &Queue) { + assert!(self + .extended_capability() + .flags() + .contains(ExtendedCapabilityFlags::QI)); + self.invalidate.queue_tail.write(0); + + let mut write_value = queue.base_paddr() as u64; + // By default, we set descriptor width to 128-bit(0) + let descriptor_width = 0b0; + write_value |= descriptor_width << 11; + + let write_queue_size = { + let mut queue_size = queue.size(); + assert!(queue_size.is_power_of_two()); + let mut write_queue_size = 0; + + if descriptor_width == 0 { + // 2^(write_queue_size + 8) = number of entries = queue_size + assert!(queue_size >= (1 << 8)); + queue_size >>= 8; + } else { + // 2^(write_queue_size + 7) = number of entries = queue_size + assert!(queue_size >= (1 << 7)); + queue_size >>= 7; + }; + + while queue_size & 0b1 == 0 { + queue_size >>= 1; + write_queue_size += 1; + } + write_queue_size + }; + + write_value |= write_queue_size; + + self.invalidate.queue_addr.write(write_value); + + // Enable Queued invalidation + self.write_global_command(GlobalCommand::QIE, true); + while !self.global_status().contains(GlobalStatus::QIES) {} + } + + fn global_invalidation(&mut self) { + // Set ICC(63) to 1 to requests invalidation and CIRG(62:61) to 01 to indicate global invalidation request. + self.context_command.write(0xA000_0000_0000_0000); + + // Wait for invalidation complete (ICC set to 0). + let mut value = 0x8000_0000_0000_0000; + while (value & 0x8000_0000_0000_0000) != 0 { + value = self.context_command.read(); + } + + // Set IVT(63) to 1 to requests IOTLB invalidation and IIRG(61:60) to 01 to indicate global invalidation request. + self.invalidate + ._iotlb_invalidate + .write(0x9000_0000_0000_0000); + } + /// Write value to the global command register. This function will not wait until the command /// is serviced. User need to check the global status register. fn write_global_command(&mut self, command: GlobalCommand, enable: bool) { @@ -156,6 +220,7 @@ impl IommuRegisters { global_status, root_table_address, context_command, + invalidate: InvalidationRegisters::new(vaddr), } }; @@ -170,7 +235,7 @@ impl IommuRegisters { } } -pub(super) static IOMMU_REGS: Once> = Once::new(); +pub(super) static IOMMU_REGS: Once> = Once::new(); pub(super) fn init() -> Result<(), IommuError> { let iommu_regs = IommuRegisters::new().ok_or(IommuError::NoIommu)?;