diff --git a/ostd/src/mm/frame/linked_list.rs b/ostd/src/mm/frame/linked_list.rs new file mode 100644 index 000000000..32ae3a10b --- /dev/null +++ b/ostd/src/mm/frame/linked_list.rs @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! Enabling linked lists of frames without heap allocation. +//! +//! This module leverages the customizability of the metadata system (see +//! [super::meta]) to allow any type of frame to be used in a linked list. + +use core::{ + ops::{Deref, DerefMut}, + ptr::NonNull, + sync::atomic::{AtomicU64, Ordering}, +}; + +use super::{ + mapping, + meta::{get_slot, AnyFrameMeta}, + unique::UniqueFrame, + MetaSlot, +}; +use crate::{ + arch::mm::PagingConsts, + mm::{Paddr, Vaddr}, + panic::abort, +}; + +/// A linked list of frames. +/// +/// Two key features that [`LinkedList`] is different from +/// [`alloc::collections::LinkedList`] is that: +/// 1. It is intrusive, meaning that the links are part of the frame metadata. +/// This allows the linked list to be used without heap allocation. But it +/// disallows a frame to be in multiple linked lists at the same time. +/// 2. The linked list exclusively own the frames, meaning that it takes +/// unique pointers [`UniqueFrame`]. And other bodies cannot +/// [`from_in_use`] a frame that is inside a linked list. +/// 3. We also allow creating cursors at a specific frame, allowing $O(1)$ +/// removal without iterating through the list at a cost of some checks. +/// +/// # Example +/// +/// To create metadata types that allows linked list links, wrap the metadata +/// type in [`Link`]: +/// +/// ```rust +/// use ostd::{ +/// mm::{frame::{linked_list::{Link, LinkedList}, Frame}, FrameAllocOptions}, +/// impl_untyped_frame_meta_for, +/// }; +/// +/// #[derive(Debug)] +/// struct MyMeta { mark: usize } +/// +/// type MyFrame = Frame>; +/// +/// impl_untyped_frame_meta_for!(MyMeta); +/// +/// let alloc_options = FrameAllocOptions::new(); +/// let frame1 = alloc_options.alloc_frame_with(Link::new(MyMeta { mark: 1 })).unwrap(); +/// let frame2 = alloc_options.alloc_frame_with(Link::new(MyMeta { mark: 2 })).unwrap(); +/// +/// let mut list = LinkedList::new(); +/// list.push_front(frame1.try_into().unwrap()); +/// list.push_front(frame2.try_into().unwrap()); +/// +/// let mut cursor = list.cursor_front_mut(); +/// assert_eq!(cursor.current_meta().unwrap().mark, 2); +/// cursor.move_next(); +/// assert_eq!(cursor.current_meta().unwrap().mark, 1); +/// ``` +/// +/// [`from_in_use`]: Frame::from_in_use +pub struct LinkedList +where + Link: AnyFrameMeta, +{ + front: Option>>, + back: Option>>, + /// The number of frames in the list. + size: usize, + /// A lazily initialized ID, used to check whether a frame is in the list. + /// 0 means uninitialized. + list_id: u64, +} + +// SAFETY: Only the pointers are not `Send` and `Sync`. But our interfaces +// enforces that only with `&mut` references can we access with the pointers. +unsafe impl Send for LinkedList where Link: AnyFrameMeta {} +unsafe impl Sync for LinkedList where Link: AnyFrameMeta {} + +impl Default for LinkedList +where + Link: AnyFrameMeta, +{ + fn default() -> Self { + Self::new() + } +} + +impl LinkedList +where + Link: AnyFrameMeta, +{ + /// Creates a new linked list. + pub const fn new() -> Self { + Self { + front: None, + back: None, + size: 0, + list_id: 0, + } + } + + /// Gets the number of frames in the linked list. + pub fn size(&self) -> usize { + self.size + } + + /// Tells if the linked list is empty. + pub fn is_empty(&self) -> bool { + let is_empty = self.size == 0; + debug_assert_eq!(is_empty, self.front.is_none()); + debug_assert_eq!(is_empty, self.back.is_none()); + is_empty + } + + /// Pushes a frame to the front of the linked list. + pub fn push_front(&mut self, frame: UniqueFrame>) { + self.cursor_front_mut().insert_before(frame); + } + + /// Pops a frame from the front of the linked list. + pub fn pop_front(&mut self) -> Option>> { + self.cursor_front_mut().take_current() + } + + /// Pushes a frame to the back of the linked list. + pub fn push_back(&mut self, frame: UniqueFrame>) { + self.cursor_at_ghost_mut().insert_before(frame); + } + + /// Pops a frame from the back of the linked list. + pub fn pop_back(&mut self) -> Option>> { + self.cursor_back_mut().take_current() + } + + /// Tells if a frame is in the list. + pub fn contains(&mut self, frame: Paddr) -> bool { + let Ok(slot) = get_slot(frame) else { + return false; + }; + slot.in_list.load(Ordering::Relaxed) == self.lazy_get_id() + } + + /// Gets a cursor at the specified frame if the frame is in the list. + /// + /// This method fail if [`Self::contains`] returns `false`. + pub fn cursor_mut_at(&mut self, frame: Paddr) -> Option> { + let Ok(slot) = get_slot(frame) else { + return None; + }; + let contains = slot.in_list.load(Ordering::Relaxed) == self.lazy_get_id(); + if contains { + Some(CursorMut { + list: self, + current: Some(NonNull::new(slot.as_meta_ptr::>()).unwrap()), + }) + } else { + None + } + } + + /// Gets a cursor at the front that can mutate the linked list links. + /// + /// If the list is empty, the cursor points to the "ghost" non-element. + pub fn cursor_front_mut(&mut self) -> CursorMut<'_, M> { + let current = self.front; + CursorMut { + list: self, + current, + } + } + + /// Gets a cursor at the back that can mutate the linked list links. + /// + /// If the list is empty, the cursor points to the "ghost" non-element. + pub fn cursor_back_mut(&mut self) -> CursorMut<'_, M> { + let current = self.back; + CursorMut { + list: self, + current, + } + } + + /// Gets a cursor at the "ghost" non-element that can mutate the linked list links. + fn cursor_at_ghost_mut(&mut self) -> CursorMut<'_, M> { + CursorMut { + list: self, + current: None, + } + } + + fn lazy_get_id(&mut self) -> u64 { + // FIXME: Self-incrementing IDs may overflow, while `core::pin::Pin` + // is not compatible with locks. Think about a better solution. + static LIST_ID_ALLOCATOR: AtomicU64 = AtomicU64::new(1); + const MAX_LIST_ID: u64 = i64::MAX as u64; + + if self.list_id == 0 { + let id = LIST_ID_ALLOCATOR.fetch_add(1, Ordering::Relaxed); + if id >= MAX_LIST_ID { + log::error!("The frame list ID allocator has exhausted."); + abort(); + } + self.list_id = id; + id + } else { + self.list_id + } + } +} + +/// A cursor that can mutate the linked list links. +/// +/// The cursor points to either a frame or the "ghost" non-element. It points +/// to the "ghost" non-element when the cursor surpasses the back of the list. +pub struct CursorMut<'a, M> +where + Link: AnyFrameMeta, +{ + list: &'a mut LinkedList, + current: Option>>, +} + +impl CursorMut<'_, M> +where + Link: AnyFrameMeta, +{ + /// Moves the cursor to the next frame towards the back. + /// + /// If the cursor is pointing to the "ghost" non-element then this will + /// move it to the first element of the [`LinkedList`]. If it is pointing + /// to the last element of the LinkedList then this will move it to the + /// "ghost" non-element. + pub fn move_next(&mut self) { + self.current = match self.current { + // SAFETY: The cursor is pointing to a valid element. + Some(current) => unsafe { current.as_ref().next }, + None => self.list.front, + }; + } + + /// Moves the cursor to the previous frame towards the front. + /// + /// If the cursor is pointing to the "ghost" non-element then this will + /// move it to the last element of the [`LinkedList`]. If it is pointing + /// to the first element of the LinkedList then this will move it to the + /// "ghost" non-element. + pub fn move_prev(&mut self) { + self.current = match self.current { + // SAFETY: The cursor is pointing to a valid element. + Some(current) => unsafe { current.as_ref().prev }, + None => self.list.back, + }; + } + + /// Gets the mutable reference to the current frame's metadata. + pub fn current_meta(&mut self) -> Option<&mut M> { + self.current.map(|current| { + // SAFETY: `&mut self` ensures we have exclusive access to the + // frame metadata. + let link_mut = unsafe { &mut *current.as_ptr() }; + // We should not allow `&mut Link` to modify the original links, + // which would break the linked list. So we just return the + // inner metadata `M`. + &mut link_mut.meta + }) + } + + /// Takes the current pointing frame out of the linked list. + /// + /// If successful, the frame is returned and the cursor is moved to the + /// next frame. If the cursor is pointing to the back of the list then it + /// is moved to the "ghost" non-element. + pub fn take_current(&mut self) -> Option>> { + let current = self.current?; + + let mut frame = { + let meta_ptr = current.as_ptr() as *mut MetaSlot; + let paddr = mapping::meta_to_frame::(meta_ptr as Vaddr); + // SAFETY: The frame was forgotten when inserted into the linked list. + unsafe { UniqueFrame::>::from_raw(paddr) } + }; + + let next_ptr = frame.meta().next; + if let Some(prev) = &mut frame.meta_mut().prev { + // SAFETY: We own the previous node by `&mut self` and the node is + // initialized. + let prev_mut = unsafe { prev.as_mut() }; + + debug_assert_eq!(prev_mut.next, Some(current)); + prev_mut.next = next_ptr; + } else { + self.list.front = next_ptr; + } + let prev_ptr = frame.meta().prev; + if let Some(next) = &mut frame.meta_mut().next { + // SAFETY: We own the next node by `&mut self` and the node is + // initialized. + let next_mut = unsafe { next.as_mut() }; + + debug_assert_eq!(next_mut.prev, Some(current)); + next_mut.prev = prev_ptr; + self.current = Some(NonNull::from(next_mut)); + } else { + self.list.back = prev_ptr; + self.current = None; + } + + frame.meta_mut().next = None; + frame.meta_mut().prev = None; + + frame.slot().in_list.store(0, Ordering::Relaxed); + + self.list.size -= 1; + + Some(frame) + } + + /// Inserts a frame before the current frame. + /// + /// If the cursor is pointing at the "ghost" non-element then the new + /// element is inserted at the back of the [`LinkedList`]. + pub fn insert_before(&mut self, mut frame: UniqueFrame>) { + // The frame can't possibly be in any linked lists since the list will + // own the frame so there can't be any unique pointers to it. + debug_assert!(frame.meta_mut().next.is_none()); + debug_assert!(frame.meta_mut().prev.is_none()); + debug_assert_eq!(frame.slot().in_list.load(Ordering::Relaxed), 0); + + let frame_ptr = NonNull::from(frame.meta_mut()); + + if let Some(current) = &mut self.current { + // SAFETY: We own the current node by `&mut self` and the node is + // initialized. + let current_mut = unsafe { current.as_mut() }; + + if let Some(prev) = &mut current_mut.prev { + // SAFETY: We own the previous node by `&mut self` and the node + // is initialized. + let prev_mut = unsafe { prev.as_mut() }; + + debug_assert_eq!(prev_mut.next, Some(*current)); + prev_mut.next = Some(frame_ptr); + + frame.meta_mut().prev = Some(*prev); + frame.meta_mut().next = Some(*current); + *prev = frame_ptr; + } else { + debug_assert_eq!(self.list.front, Some(*current)); + frame.meta_mut().next = Some(*current); + current_mut.prev = Some(frame_ptr); + self.list.front = Some(frame_ptr); + } + } else { + // We are at the "ghost" non-element. + if let Some(back) = &mut self.list.back { + // SAFETY: We have ownership of the links via `&mut self`. + unsafe { + debug_assert!(back.as_mut().next.is_none()); + back.as_mut().next = Some(frame_ptr); + } + frame.meta_mut().prev = Some(*back); + self.list.back = Some(frame_ptr); + } else { + debug_assert_eq!(self.list.front, None); + self.list.front = Some(frame_ptr); + self.list.back = Some(frame_ptr); + } + } + + frame + .slot() + .in_list + .store(self.list.lazy_get_id(), Ordering::Relaxed); + + // Forget the frame to transfer the ownership to the list. + let _ = frame.into_raw(); + + self.list.size += 1; + } + + /// Provides a reference to the linked list. + pub fn as_list(&self) -> &LinkedList { + &*self.list + } +} + +impl Drop for LinkedList +where + Link: AnyFrameMeta, +{ + fn drop(&mut self) { + let mut cursor = self.cursor_front_mut(); + while cursor.take_current().is_some() {} + } +} + +/// The metadata of linked list frames. +/// +/// To allow other metadata to be customized, this type is a wrapper around the +/// actual metadata type `M`. +/// +/// Linked list frames can be contained in a [`LinkedList`]. +#[derive(Debug)] +pub struct Link { + next: Option>>, + prev: Option>>, + meta: M, +} + +// SAFETY: `Link` is `Send` and `Sync` if `M` is `Send` and `Sync` because +// we only access these unsafe cells when the frame is not shared. This is +// enforced by `UniqueFrame`. +unsafe impl Send for Link {} +unsafe impl Sync for Link {} + +impl Deref for Link { + type Target = M; + + fn deref(&self) -> &Self::Target { + &self.meta + } +} + +impl DerefMut for Link { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.meta + } +} + +impl Link { + /// Creates a new linked list metadata. + pub const fn new(meta: M) -> Self { + Self { + next: None, + prev: None, + meta, + } + } +} + +// SAFETY: The size and alignment of `Link` must be within the limits. +// Also, if `M` is typed, `Link` must not be untyped. +// +// FIXME: We would appreciate constant trait bounds very much, just like: +// ``` +// Assert<{ core::mem::size_of::>() < FRAME_METADATA_MAX_SIZE }>: IsTrue, +// Assert<{ core::mem::align_of::>() <= FRAME_METADATA_MAX_ALIGN }>: IsTrue, +// ``` +// But due to the broken implementation of `generic_const_exprs` we can't do that. +// It works for `M` that is declared inside OSTD but not for outside types. +unsafe impl AnyFrameMeta for Link +where + M: AnyFrameMeta, +{ + fn on_drop(&mut self, reader: &mut crate::mm::VmReader) { + self.meta.on_drop(reader); + } + + fn is_untyped(&self) -> bool { + self.meta.is_untyped() + } +} + +#[cfg(ktest)] +mod test { + use super::*; + use crate::{impl_untyped_frame_meta_for, mm::FrameAllocOptions, prelude::ktest}; + + #[derive(Debug)] + struct MyMeta { + mark: usize, + } + + impl_untyped_frame_meta_for!(MyMeta); + + #[ktest] + fn linked_list_push_pop() { + let alloc_options = FrameAllocOptions::new(); + let frame1 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 1 })) + .unwrap(); + let frame2 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 2 })) + .unwrap(); + let frame3 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 3 })) + .unwrap(); + + let mut list = LinkedList::new(); + assert!(list.is_empty()); + list.push_front(frame1.try_into().unwrap()); + // 1 + list.push_front(frame2.try_into().unwrap()); + // 2 1 + list.push_back(frame3.try_into().unwrap()); + // 2 1 3 + assert_eq!(list.size(), 3); + + let mut cursor = list.cursor_front_mut(); + // *2 1 3 () + assert_eq!(cursor.current_meta().unwrap().mark, 2); + cursor.move_next(); + // 2 *1 3 () + assert_eq!(cursor.current_meta().unwrap().mark, 1); + cursor.move_next(); + // 2 1 *3 () + assert_eq!(cursor.current_meta().unwrap().mark, 3); + cursor.move_next(); + // 2 1 3 *() + assert!(cursor.current_meta().is_none()); + + assert_eq!(cursor.as_list().size(), 3); + + assert_eq!(list.pop_front().unwrap().meta().mark, 2); + assert_eq!(list.size(), 2); + // 1 3 + assert_eq!(list.pop_back().unwrap().meta().mark, 3); + assert_eq!(list.size(), 1); + // 1 + assert_eq!(list.pop_front().unwrap().meta().mark, 1); + assert_eq!(list.size(), 0); + } + + #[ktest] + fn linked_list_cursor_at() { + let alloc_options = FrameAllocOptions::new(); + let frame1 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 1 })) + .unwrap(); + let frame1_addr = frame1.start_paddr(); + let frame2 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 2 })) + .unwrap(); + let frame2_addr = frame2.start_paddr(); + let frame3 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 3 })) + .unwrap(); + let frame3_addr = frame3.start_paddr(); + + let frame_outside = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 4 })) + .unwrap(); + + let mut list = LinkedList::new(); + list.push_front(frame1.try_into().unwrap()); + list.push_front(frame2.try_into().unwrap()); + list.push_front(frame3.try_into().unwrap()); + + assert!(!list.contains(frame_outside.start_paddr())); + assert!(list.cursor_mut_at(frame_outside.start_paddr()).is_none()); + + assert!(list.contains(frame1_addr)); + assert!(list.contains(frame2_addr)); + assert!(list.contains(frame3_addr)); + + let mut cursor = list.cursor_mut_at(frame1_addr).unwrap(); + assert_eq!(cursor.current_meta().unwrap().mark, 1); + let mut cursor = list.cursor_mut_at(frame2_addr).unwrap(); + assert_eq!(cursor.current_meta().unwrap().mark, 2); + let mut cursor = list.cursor_mut_at(frame3_addr).unwrap(); + assert_eq!(cursor.current_meta().unwrap().mark, 3); + } + + #[ktest] + fn linked_list_cursor_ops() { + let alloc_options = FrameAllocOptions::new(); + let frame1 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 1 })) + .unwrap(); + let frame2 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 2 })) + .unwrap(); + let frame3 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 3 })) + .unwrap(); + let frame4 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 4 })) + .unwrap(); + let frame5 = alloc_options + .alloc_frame_with(Link::new(MyMeta { mark: 5 })) + .unwrap(); + + let mut list = LinkedList::new(); + assert!(list.is_empty()); + list.push_front(frame1.try_into().unwrap()); + // 1 *() + list.push_front(frame2.try_into().unwrap()); + // 2 *1 () + assert_eq!(list.size(), 2); + + let mut cursor = list.cursor_front_mut(); + // *2 1 () + assert_eq!(cursor.current_meta().unwrap().mark, 2); + cursor.move_next(); + // 2 *1 () + assert_eq!(cursor.current_meta().unwrap().mark, 1); + cursor.insert_before(frame3.try_into().unwrap()); + // 2 3 *1 () + assert_eq!(cursor.current_meta().unwrap().mark, 1); + cursor.insert_before(frame4.try_into().unwrap()); + // 2 3 4 *1 () + assert_eq!(cursor.current_meta().unwrap().mark, 1); + cursor.move_next(); + // 2 3 4 1 *() + assert!(cursor.current_meta().is_none()); + cursor.insert_before(frame5.try_into().unwrap()); + // 2 3 4 1 5 *() + assert!(cursor.current_meta().is_none()); + + assert_eq!(cursor.as_list().size(), 5); + + let mut cursor = list.cursor_front_mut(); + assert_eq!(cursor.current_meta().unwrap().mark, 2); + cursor.move_next(); + assert_eq!(cursor.current_meta().unwrap().mark, 3); + cursor.move_next(); + assert_eq!(cursor.current_meta().unwrap().mark, 4); + cursor.move_next(); + assert_eq!(cursor.current_meta().unwrap().mark, 1); + cursor.move_next(); + assert_eq!(cursor.current_meta().unwrap().mark, 5); + cursor.move_next(); + assert!(cursor.current_meta().is_none()); + // 2 3 4 1 5 *() + + cursor.move_prev(); + // 2 3 4 1 *5 () + assert_eq!(cursor.current_meta().unwrap().mark, 5); + cursor.move_prev(); + // 2 3 4 *1 5 () + assert_eq!(cursor.current_meta().unwrap().mark, 1); + + let frame1 = cursor.take_current().unwrap(); + // 2 3 4 *5 () + assert_eq!(frame1.meta().mark, 1); + assert_eq!(cursor.current_meta().unwrap().mark, 5); + cursor.move_next(); + // 2 3 4 5 *() + assert!(cursor.current_meta().is_none()); + cursor.move_prev(); + // 2 3 4 *5 () + assert_eq!(cursor.current_meta().unwrap().mark, 5); + cursor.move_prev(); + // 2 3 *4 5 () + assert_eq!(cursor.current_meta().unwrap().mark, 4); + cursor.move_prev(); + // 2 *3 4 5 () + assert_eq!(cursor.current_meta().unwrap().mark, 3); + cursor.move_prev(); + // *2 3 4 5 () + assert_eq!(cursor.current_meta().unwrap().mark, 2); + + let frame2 = cursor.take_current().unwrap(); + // *3 4 5 () + assert_eq!(frame2.meta().mark, 2); + assert_eq!(cursor.current_meta().unwrap().mark, 3); + cursor.move_next(); + // 3 *4 5 () + assert_eq!(cursor.current_meta().unwrap().mark, 4); + cursor.move_next(); + // 3 4 *5 () + assert_eq!(cursor.current_meta().unwrap().mark, 5); + // 3 4 *() + let frame5 = cursor.take_current().unwrap(); + assert_eq!(frame5.meta().mark, 5); + assert!(cursor.current_meta().is_none()); + + assert_eq!(cursor.as_list().size(), 2); + } +} diff --git a/ostd/src/mm/frame/meta.rs b/ostd/src/mm/frame/meta.rs index b2246220c..82251ba59 100644 --- a/ostd/src/mm/frame/meta.rs +++ b/ostd/src/mm/frame/meta.rs @@ -63,8 +63,10 @@ use crate::{ }; /// The maximum number of bytes of the metadata of a frame. -pub const FRAME_METADATA_MAX_SIZE: usize = - META_SLOT_SIZE - size_of::() - size_of::(); +pub const FRAME_METADATA_MAX_SIZE: usize = META_SLOT_SIZE + - size_of::() + - size_of::() + - size_of::(); /// The maximum alignment in bytes of the metadata of a frame. pub const FRAME_METADATA_MAX_ALIGN: usize = META_SLOT_SIZE; @@ -105,6 +107,15 @@ pub(in crate::mm) struct MetaSlot { pub(super) ref_count: AtomicU64, /// The virtual table that indicates the type of the metadata. pub(super) vtable_ptr: UnsafeCell>, + /// This is only accessed by [`crate::mm::frame::linked_list`]. + /// It stores 0 if the frame is not in any list, otherwise it stores the + /// ID of the list. + /// + /// It is ugly but allows us to tell if a frame is in a specific list by + /// one relaxed read. Otherwise, if we store it conditionally in `storage` + /// we would have to ensure that the type is correct before the read, which + /// costs a synchronization. + pub(super) in_list: AtomicU64, } pub(super) const REF_COUNT_UNUSED: u64 = u64::MAX; @@ -315,8 +326,8 @@ impl MetaSlot { /// The returned pointer should not be dereferenced as mutable unless having /// exclusive access to the metadata slot. pub(super) unsafe fn dyn_meta_ptr(&self) -> *mut dyn AnyFrameMeta { - // SAFETY: The page metadata is valid to be borrowed mutably, since it will never be - // borrowed immutably after initialization. + // SAFETY: The page metadata is valid to be borrowed immutably, since + // it will never be borrowed mutably after initialization. let vtable_ptr = unsafe { *self.vtable_ptr.get() }; // SAFETY: The page metadata is initialized and valid. @@ -498,18 +509,21 @@ fn alloc_meta_frames(tot_nr_frames: usize) -> (usize, Paddr) { let slots = paddr_to_vaddr(start_paddr) as *mut MetaSlot; - // Fill the metadata frames with a byte pattern of `REF_COUNT_UNUSED`. - debug_assert_eq!(REF_COUNT_UNUSED.to_ne_bytes(), [0xff; 8]); - // SAFETY: `slots` and the length is a valid region for the metadata frames - // that are going to be treated as metadata slots. The byte pattern is - // valid as the initial value of the reference count (other fields are - // either not accessed or `MaybeUninit`). - unsafe { - core::ptr::write_bytes( - slots as *mut u8, - 0xff, - tot_nr_frames * size_of::(), - ); + // Initialize the metadata slots. + for i in 0..tot_nr_frames { + // SAFETY: The memory is successfully allocated with `tot_nr_frames` + // slots so the index must be within the range. + let slot = unsafe { slots.add(i) }; + // SAFETY: The memory is just allocated so we have exclusive access and + // it's valid for writing. + unsafe { + slot.write(MetaSlot { + storage: UnsafeCell::new([0; FRAME_METADATA_MAX_SIZE]), + ref_count: AtomicU64::new(REF_COUNT_UNUSED), + vtable_ptr: UnsafeCell::new(MaybeUninit::uninit()), + in_list: AtomicU64::new(0), + }) + }; } (nr_meta_pages, start_paddr) diff --git a/ostd/src/mm/frame/mod.rs b/ostd/src/mm/frame/mod.rs index 2c8eba778..d9f2f3900 100644 --- a/ostd/src/mm/frame/mod.rs +++ b/ostd/src/mm/frame/mod.rs @@ -32,6 +32,7 @@ //! can create custom metadata types by implementing the [`AnyFrameMeta`] trait. pub mod allocator; +pub mod linked_list; pub mod meta; pub mod segment; pub mod unique; @@ -162,9 +163,8 @@ impl Frame { /// restored using [`Frame::from_raw`] later. This is useful when some architectural /// data structures need to hold the frame handle such as the page table. pub(in crate::mm) fn into_raw(self) -> Paddr { - let paddr = self.start_paddr(); - let _ = ManuallyDrop::new(self); - paddr + let this = ManuallyDrop::new(self); + this.start_paddr() } /// Restores a forgotten [`Frame`] from a physical address. diff --git a/ostd/src/mm/frame/unique.rs b/ostd/src/mm/frame/unique.rs index 0fe785d79..e2e1a6041 100644 --- a/ostd/src/mm/frame/unique.rs +++ b/ostd/src/mm/frame/unique.rs @@ -2,13 +2,13 @@ //! The unique frame pointer that is not shared with others. -use core::{marker::PhantomData, sync::atomic::Ordering}; +use core::{marker::PhantomData, mem::ManuallyDrop, sync::atomic::Ordering}; use super::{ meta::{GetFrameError, REF_COUNT_UNIQUE}, AnyFrameMeta, Frame, MetaSlot, }; -use crate::mm::{Paddr, PagingLevel, PAGE_SIZE}; +use crate::mm::{frame::mapping, Paddr, PagingConsts, PagingLevel, PAGE_SIZE}; /// An owning frame pointer. /// @@ -99,7 +99,29 @@ impl UniqueFrame { unsafe { &mut *self.slot().dyn_meta_ptr() } } - fn slot(&self) -> &MetaSlot { + /// Converts this frame into a raw physical address. + pub(crate) fn into_raw(self) -> Paddr { + let this = ManuallyDrop::new(self); + this.start_paddr() + } + + /// Restores a raw physical address back into a unique frame. + /// + /// # Safety + /// + /// The caller must ensure that the physical address is valid and points to + /// a forgotten frame that was previously casted by [`Self::into_raw`]. + pub(crate) unsafe fn from_raw(paddr: Paddr) -> Self { + let vaddr = mapping::frame_to_meta::(paddr); + let ptr = vaddr as *const MetaSlot; + + Self { + ptr, + _marker: PhantomData, + } + } + + pub(super) fn slot(&self) -> &MetaSlot { // SAFETY: `ptr` points to a valid `MetaSlot` that will never be // mutably borrowed, so taking an immutable reference to it is safe. unsafe { &*self.ptr }