Linked lists of Frames

ostd/src/mm/frame/linked_list.rs

@@ -0,0 +1,681 @@
+// SPDX-License-Identifier: MPL-2.0
+
+//! Enabling linked lists of frames without heap allocation.
+//!
+//! This module leverages the customizability of the metadata system (see
+//! [super::meta]) to allow any type of frame to be used in a linked list.
+
+use core::{
+    ops::{Deref, DerefMut},
+    ptr::NonNull,
+    sync::atomic::{AtomicU64, Ordering},
+};
+
+use super::{
+    mapping,
+    meta::{get_slot, AnyFrameMeta},
+    unique::UniqueFrame,
+    MetaSlot,
+};
+use crate::{
+    arch::mm::PagingConsts,
+    mm::{Paddr, Vaddr},
+    panic::abort,
+};
+
+/// A linked list of frames.
+///
+/// Two key features that [`LinkedList`] is different from
+/// [`alloc::collections::LinkedList`] is that:
+///  1. It is intrusive, meaning that the links are part of the frame metadata.
+///     This allows the linked list to be used without heap allocation. But it
+///     disallows a frame to be in multiple linked lists at the same time.
+///  2. The linked list exclusively own the frames, meaning that it takes
+///     unique pointers [`UniqueFrame`]. And other bodies cannot
+///     [`from_in_use`] a frame that is inside a linked list.
+///  3. We also allow creating cursors at a specific frame, allowing $O(1)$
+///     removal without iterating through the list at a cost of some checks.
+///
+/// # Example
+///
+/// To create metadata types that allows linked list links, wrap the metadata
+/// type in [`Link`]:
+///
+/// ```rust
+/// use ostd::{
+///     mm::{frame::{linked_list::{Link, LinkedList}, Frame}, FrameAllocOptions},
+///     impl_untyped_frame_meta_for,
+/// };
+///
+/// #[derive(Debug)]
+/// struct MyMeta { mark: usize }
+///
+/// type MyFrame = Frame<Link<MyMeta>>;
+///
+/// impl_untyped_frame_meta_for!(MyMeta);
+///
+/// let alloc_options = FrameAllocOptions::new();
+/// let frame1 = alloc_options.alloc_frame_with(Link::new(MyMeta { mark: 1 })).unwrap();
+/// let frame2 = alloc_options.alloc_frame_with(Link::new(MyMeta { mark: 2 })).unwrap();
+///
+/// let mut list = LinkedList::new();
+/// list.push_front(frame1.try_into().unwrap());
+/// list.push_front(frame2.try_into().unwrap());
+///
+/// let mut cursor = list.cursor_front_mut();
+/// assert_eq!(cursor.current_meta().unwrap().mark, 2);
+/// cursor.move_next();
+/// assert_eq!(cursor.current_meta().unwrap().mark, 1);
+/// ```
+///
+/// [`from_in_use`]: Frame::from_in_use
+pub struct LinkedList<M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    front: Option<NonNull<Link<M>>>,
+    back: Option<NonNull<Link<M>>>,
+    /// The number of frames in the list.
+    size: usize,
+    /// A lazily initialized ID, used to check whether a frame is in the list.
+    /// 0 means uninitialized.
+    list_id: u64,
+}
+
+// SAFETY: Only the pointers are not `Send` and `Sync`. But our interfaces
+// enforces that only with `&mut` references can we access with the pointers.
+unsafe impl<M> Send for LinkedList<M> where Link<M>: AnyFrameMeta {}
+unsafe impl<M> Sync for LinkedList<M> where Link<M>: AnyFrameMeta {}
+
+impl<M> Default for LinkedList<M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<M> LinkedList<M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    /// Creates a new linked list.
+    pub const fn new() -> Self {
+        Self {
+            front: None,
+            back: None,
+            size: 0,
+            list_id: 0,
+        }
+    }
+
+    /// Gets the number of frames in the linked list.
+    pub fn size(&self) -> usize {
+        self.size
+    }
+
+    /// Tells if the linked list is empty.
+    pub fn is_empty(&self) -> bool {
+        let is_empty = self.size == 0;
+        debug_assert_eq!(is_empty, self.front.is_none());
+        debug_assert_eq!(is_empty, self.back.is_none());
+        is_empty
+    }
+
+    /// Pushes a frame to the front of the linked list.
+    pub fn push_front(&mut self, frame: UniqueFrame<Link<M>>) {
+        self.cursor_front_mut().insert_before(frame);
+    }
+
+    /// Pops a frame from the front of the linked list.
+    pub fn pop_front(&mut self) -> Option<UniqueFrame<Link<M>>> {
+        self.cursor_front_mut().take_current()
+    }
+
+    /// Pushes a frame to the back of the linked list.
+    pub fn push_back(&mut self, frame: UniqueFrame<Link<M>>) {
+        self.cursor_at_ghost_mut().insert_before(frame);
+    }
+
+    /// Pops a frame from the back of the linked list.
+    pub fn pop_back(&mut self) -> Option<UniqueFrame<Link<M>>> {
+        self.cursor_back_mut().take_current()
+    }
+
+    /// Tells if a frame is in the list.
+    pub fn contains(&mut self, frame: Paddr) -> bool {
+        let Ok(slot) = get_slot(frame) else {
+            return false;
+        };
+        slot.in_list.load(Ordering::Relaxed) == self.lazy_get_id()
+    }
+
+    /// Gets a cursor at the specified frame if the frame is in the list.
+    ///
+    /// This method fail if [`Self::contains`] returns `false`.
+    pub fn cursor_mut_at(&mut self, frame: Paddr) -> Option<CursorMut<'_, M>> {
+        let Ok(slot) = get_slot(frame) else {
+            return None;
+        };
+        let contains = slot.in_list.load(Ordering::Relaxed) == self.lazy_get_id();
+        if contains {
+            Some(CursorMut {
+                list: self,
+                current: Some(NonNull::new(slot.as_meta_ptr::<Link<M>>()).unwrap()),
+            })
+        } else {
+            None
+        }
+    }
+
+    /// Gets a cursor at the front that can mutate the linked list links.
+    ///
+    /// If the list is empty, the cursor points to the "ghost" non-element.
+    pub fn cursor_front_mut(&mut self) -> CursorMut<'_, M> {
+        let current = self.front;
+        CursorMut {
+            list: self,
+            current,
+        }
+    }
+
+    /// Gets a cursor at the back that can mutate the linked list links.
+    ///
+    /// If the list is empty, the cursor points to the "ghost" non-element.
+    pub fn cursor_back_mut(&mut self) -> CursorMut<'_, M> {
+        let current = self.back;
+        CursorMut {
+            list: self,
+            current,
+        }
+    }
+
+    /// Gets a cursor at the "ghost" non-element that can mutate the linked list links.
+    fn cursor_at_ghost_mut(&mut self) -> CursorMut<'_, M> {
+        CursorMut {
+            list: self,
+            current: None,
+        }
+    }
+
+    fn lazy_get_id(&mut self) -> u64 {
+        // FIXME: Self-incrementing IDs may overflow, while `core::pin::Pin`
+        // is not compatible with locks. Think about a better solution.
+        static LIST_ID_ALLOCATOR: AtomicU64 = AtomicU64::new(1);
+        const MAX_LIST_ID: u64 = i64::MAX as u64;
+
+        if self.list_id == 0 {
+            let id = LIST_ID_ALLOCATOR.fetch_add(1, Ordering::Relaxed);
+            if id >= MAX_LIST_ID {
+                log::error!("The frame list ID allocator has exhausted.");
+                abort();
+            }
+            self.list_id = id;
+            id
+        } else {
+            self.list_id
+        }
+    }
+}
+
+/// A cursor that can mutate the linked list links.
+///
+/// The cursor points to either a frame or the "ghost" non-element. It points
+/// to the "ghost" non-element when the cursor surpasses the back of the list.
+pub struct CursorMut<'a, M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    list: &'a mut LinkedList<M>,
+    current: Option<NonNull<Link<M>>>,
+}
+
+impl<M> CursorMut<'_, M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    /// Moves the cursor to the next frame towards the back.
+    ///
+    /// If the cursor is pointing to the "ghost" non-element then this will
+    /// move it to the first element of the [`LinkedList`]. If it is pointing
+    /// to the last element of the LinkedList then this will move it to the
+    /// "ghost" non-element.
+    pub fn move_next(&mut self) {
+        self.current = match self.current {
+            // SAFETY: The cursor is pointing to a valid element.
+            Some(current) => unsafe { current.as_ref().next },
+            None => self.list.front,
+        };
+    }
+
+    /// Moves the cursor to the previous frame towards the front.
+    ///
+    /// If the cursor is pointing to the "ghost" non-element then this will
+    /// move it to the last element of the [`LinkedList`]. If it is pointing
+    /// to the first element of the LinkedList then this will move it to the
+    /// "ghost" non-element.
+    pub fn move_prev(&mut self) {
+        self.current = match self.current {
+            // SAFETY: The cursor is pointing to a valid element.
+            Some(current) => unsafe { current.as_ref().prev },
+            None => self.list.back,
+        };
+    }
+
+    /// Gets the mutable reference to the current frame's metadata.
+    pub fn current_meta(&mut self) -> Option<&mut M> {
+        self.current.map(|current| {
+            // SAFETY: `&mut self` ensures we have exclusive access to the
+            // frame metadata.
+            let link_mut = unsafe { &mut *current.as_ptr() };
+            // We should not allow `&mut Link<M>` to modify the original links,
+            // which would break the linked list. So we just return the
+            // inner metadata `M`.
+            &mut link_mut.meta
+        })
+    }
+
+    /// Takes the current pointing frame out of the linked list.
+    ///
+    /// If successful, the frame is returned and the cursor is moved to the
+    /// next frame. If the cursor is pointing to the back of the list then it
+    /// is moved to the "ghost" non-element.
+    pub fn take_current(&mut self) -> Option<UniqueFrame<Link<M>>> {
+        let current = self.current?;
+
+        let mut frame = {
+            let meta_ptr = current.as_ptr() as *mut MetaSlot;
+            let paddr = mapping::meta_to_frame::<PagingConsts>(meta_ptr as Vaddr);
+            // SAFETY: The frame was forgotten when inserted into the linked list.
+            unsafe { UniqueFrame::<Link<M>>::from_raw(paddr) }
+        };
+
+        let next_ptr = frame.meta().next;
+        if let Some(prev) = &mut frame.meta_mut().prev {
+            // SAFETY: We own the previous node by `&mut self` and the node is
+            // initialized.
+            let prev_mut = unsafe { prev.as_mut() };
+
+            debug_assert_eq!(prev_mut.next, Some(current));
+            prev_mut.next = next_ptr;
+        } else {
+            self.list.front = next_ptr;
+        }
+        let prev_ptr = frame.meta().prev;
+        if let Some(next) = &mut frame.meta_mut().next {
+            // SAFETY: We own the next node by `&mut self` and the node is
+            // initialized.
+            let next_mut = unsafe { next.as_mut() };
+
+            debug_assert_eq!(next_mut.prev, Some(current));
+            next_mut.prev = prev_ptr;
+            self.current = Some(NonNull::from(next_mut));
+        } else {
+            self.list.back = prev_ptr;
+            self.current = None;
+        }
+
+        frame.meta_mut().next = None;
+        frame.meta_mut().prev = None;
+
+        frame.slot().in_list.store(0, Ordering::Relaxed);
+
+        self.list.size -= 1;
+
+        Some(frame)
+    }
+
+    /// Inserts a frame before the current frame.
+    ///
+    /// If the cursor is pointing at the "ghost" non-element then the new
+    /// element is inserted at the back of the [`LinkedList`].
+    pub fn insert_before(&mut self, mut frame: UniqueFrame<Link<M>>) {
+        // The frame can't possibly be in any linked lists since the list will
+        // own the frame so there can't be any unique pointers to it.
+        debug_assert!(frame.meta_mut().next.is_none());
+        debug_assert!(frame.meta_mut().prev.is_none());
+        debug_assert_eq!(frame.slot().in_list.load(Ordering::Relaxed), 0);
+
+        let frame_ptr = NonNull::from(frame.meta_mut());
+
+        if let Some(current) = &mut self.current {
+            // SAFETY: We own the current node by `&mut self` and the node is
+            // initialized.
+            let current_mut = unsafe { current.as_mut() };
+
+            if let Some(prev) = &mut current_mut.prev {
+                // SAFETY: We own the previous node by `&mut self` and the node
+                // is initialized.
+                let prev_mut = unsafe { prev.as_mut() };
+
+                debug_assert_eq!(prev_mut.next, Some(*current));
+                prev_mut.next = Some(frame_ptr);
+
+                frame.meta_mut().prev = Some(*prev);
+                frame.meta_mut().next = Some(*current);
+                *prev = frame_ptr;
+            } else {
+                debug_assert_eq!(self.list.front, Some(*current));
+                frame.meta_mut().next = Some(*current);
+                current_mut.prev = Some(frame_ptr);
+                self.list.front = Some(frame_ptr);
+            }
+        } else {
+            // We are at the "ghost" non-element.
+            if let Some(back) = &mut self.list.back {
+                // SAFETY: We have ownership of the links via `&mut self`.
+                unsafe {
+                    debug_assert!(back.as_mut().next.is_none());
+                    back.as_mut().next = Some(frame_ptr);
+                }
+                frame.meta_mut().prev = Some(*back);
+                self.list.back = Some(frame_ptr);
+            } else {
+                debug_assert_eq!(self.list.front, None);
+                self.list.front = Some(frame_ptr);
+                self.list.back = Some(frame_ptr);
+            }
+        }
+
+        frame
+            .slot()
+            .in_list
+            .store(self.list.lazy_get_id(), Ordering::Relaxed);
+
+        // Forget the frame to transfer the ownership to the list.
+        let _ = frame.into_raw();
+
+        self.list.size += 1;
+    }
+
+    /// Provides a reference to the linked list.
+    pub fn as_list(&self) -> &LinkedList<M> {
+        &*self.list
+    }
+}
+
+impl<M> Drop for LinkedList<M>
+where
+    Link<M>: AnyFrameMeta,
+{
+    fn drop(&mut self) {
+        let mut cursor = self.cursor_front_mut();
+        while cursor.take_current().is_some() {}
+    }
+}
+
+/// The metadata of linked list frames.
+///
+/// To allow other metadata to be customized, this type is a wrapper around the
+/// actual metadata type `M`.
+///
+/// Linked list frames can be contained in a [`LinkedList`].
+#[derive(Debug)]
+pub struct Link<M> {
+    next: Option<NonNull<Link<M>>>,
+    prev: Option<NonNull<Link<M>>>,
+    meta: M,
+}
+
+// SAFETY: `Link<M>` is `Send` and `Sync` if `M` is `Send` and `Sync` because
+// we only access these unsafe cells when the frame is not shared. This is
+// enforced by `UniqueFrame`.
+unsafe impl<M: Send> Send for Link<M> {}
+unsafe impl<M: Sync> Sync for Link<M> {}
+
+impl<M> Deref for Link<M> {
+    type Target = M;
+
+    fn deref(&self) -> &Self::Target {
+        &self.meta
+    }
+}
+
+impl<M> DerefMut for Link<M> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.meta
+    }
+}
+
+impl<M> Link<M> {
+    /// Creates a new linked list metadata.
+    pub const fn new(meta: M) -> Self {
+        Self {
+            next: None,
+            prev: None,
+            meta,
+        }
+    }
+}
+
+// SAFETY: The size and alignment of `Link<M>` must be within the limits.
+// Also, if `M` is typed, `Link<M>` must not be untyped.
+//
+// FIXME: We would appreciate constant trait bounds very much, just like:
+// ```
+// Assert<{ core::mem::size_of::<Link<M>>() < FRAME_METADATA_MAX_SIZE }>: IsTrue,
+// Assert<{ core::mem::align_of::<Link<M>>() <= FRAME_METADATA_MAX_ALIGN }>: IsTrue,
+// ```
+// But due to the broken implementation of `generic_const_exprs` we can't do that.
+// It works for `M` that is declared inside OSTD but not for outside types.
+unsafe impl<M> AnyFrameMeta for Link<M>
+where
+    M: AnyFrameMeta,
+{
+    fn on_drop(&mut self, reader: &mut crate::mm::VmReader<crate::mm::Infallible>) {
+        self.meta.on_drop(reader);
+    }
+
+    fn is_untyped(&self) -> bool {
+        self.meta.is_untyped()
+    }
+}
+
+#[cfg(ktest)]
+mod test {
+    use super::*;
+    use crate::{impl_untyped_frame_meta_for, mm::FrameAllocOptions, prelude::ktest};
+
+    #[derive(Debug)]
+    struct MyMeta {
+        mark: usize,
+    }
+
+    impl_untyped_frame_meta_for!(MyMeta);
+
+    #[ktest]
+    fn linked_list_push_pop() {
+        let alloc_options = FrameAllocOptions::new();
+        let frame1 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 1 }))
+            .unwrap();
+        let frame2 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 2 }))
+            .unwrap();
+        let frame3 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 3 }))
+            .unwrap();
+
+        let mut list = LinkedList::new();
+        assert!(list.is_empty());
+        list.push_front(frame1.try_into().unwrap());
+        // 1
+        list.push_front(frame2.try_into().unwrap());
+        // 2 1
+        list.push_back(frame3.try_into().unwrap());
+        // 2 1 3
+        assert_eq!(list.size(), 3);
+
+        let mut cursor = list.cursor_front_mut();
+        // *2 1 3 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 2);
+        cursor.move_next();
+        // 2 *1 3 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        cursor.move_next();
+        // 2 1 *3 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 3);
+        cursor.move_next();
+        // 2 1 3 *()
+        assert!(cursor.current_meta().is_none());
+
+        assert_eq!(cursor.as_list().size(), 3);
+
+        assert_eq!(list.pop_front().unwrap().meta().mark, 2);
+        assert_eq!(list.size(), 2);
+        // 1 3
+        assert_eq!(list.pop_back().unwrap().meta().mark, 3);
+        assert_eq!(list.size(), 1);
+        // 1
+        assert_eq!(list.pop_front().unwrap().meta().mark, 1);
+        assert_eq!(list.size(), 0);
+    }
+
+    #[ktest]
+    fn linked_list_cursor_at() {
+        let alloc_options = FrameAllocOptions::new();
+        let frame1 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 1 }))
+            .unwrap();
+        let frame1_addr = frame1.start_paddr();
+        let frame2 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 2 }))
+            .unwrap();
+        let frame2_addr = frame2.start_paddr();
+        let frame3 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 3 }))
+            .unwrap();
+        let frame3_addr = frame3.start_paddr();
+
+        let frame_outside = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 4 }))
+            .unwrap();
+
+        let mut list = LinkedList::new();
+        list.push_front(frame1.try_into().unwrap());
+        list.push_front(frame2.try_into().unwrap());
+        list.push_front(frame3.try_into().unwrap());
+
+        assert!(!list.contains(frame_outside.start_paddr()));
+        assert!(list.cursor_mut_at(frame_outside.start_paddr()).is_none());
+
+        assert!(list.contains(frame1_addr));
+        assert!(list.contains(frame2_addr));
+        assert!(list.contains(frame3_addr));
+
+        let mut cursor = list.cursor_mut_at(frame1_addr).unwrap();
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        let mut cursor = list.cursor_mut_at(frame2_addr).unwrap();
+        assert_eq!(cursor.current_meta().unwrap().mark, 2);
+        let mut cursor = list.cursor_mut_at(frame3_addr).unwrap();
+        assert_eq!(cursor.current_meta().unwrap().mark, 3);
+    }
+
+    #[ktest]
+    fn linked_list_cursor_ops() {
+        let alloc_options = FrameAllocOptions::new();
+        let frame1 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 1 }))
+            .unwrap();
+        let frame2 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 2 }))
+            .unwrap();
+        let frame3 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 3 }))
+            .unwrap();
+        let frame4 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 4 }))
+            .unwrap();
+        let frame5 = alloc_options
+            .alloc_frame_with(Link::new(MyMeta { mark: 5 }))
+            .unwrap();
+
+        let mut list = LinkedList::new();
+        assert!(list.is_empty());
+        list.push_front(frame1.try_into().unwrap());
+        // 1 *()
+        list.push_front(frame2.try_into().unwrap());
+        // 2 *1 ()
+        assert_eq!(list.size(), 2);
+
+        let mut cursor = list.cursor_front_mut();
+        // *2 1 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 2);
+        cursor.move_next();
+        // 2 *1 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        cursor.insert_before(frame3.try_into().unwrap());
+        // 2 3 *1 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        cursor.insert_before(frame4.try_into().unwrap());
+        // 2 3 4 *1 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        cursor.move_next();
+        // 2 3 4 1 *()
+        assert!(cursor.current_meta().is_none());
+        cursor.insert_before(frame5.try_into().unwrap());
+        // 2 3 4 1 5 *()
+        assert!(cursor.current_meta().is_none());
+
+        assert_eq!(cursor.as_list().size(), 5);
+
+        let mut cursor = list.cursor_front_mut();
+        assert_eq!(cursor.current_meta().unwrap().mark, 2);
+        cursor.move_next();
+        assert_eq!(cursor.current_meta().unwrap().mark, 3);
+        cursor.move_next();
+        assert_eq!(cursor.current_meta().unwrap().mark, 4);
+        cursor.move_next();
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+        cursor.move_next();
+        assert_eq!(cursor.current_meta().unwrap().mark, 5);
+        cursor.move_next();
+        assert!(cursor.current_meta().is_none());
+        // 2 3 4 1 5 *()
+
+        cursor.move_prev();
+        // 2 3 4 1 *5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 5);
+        cursor.move_prev();
+        // 2 3 4 *1 5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 1);
+
+        let frame1 = cursor.take_current().unwrap();
+        // 2 3 4 *5 ()
+        assert_eq!(frame1.meta().mark, 1);
+        assert_eq!(cursor.current_meta().unwrap().mark, 5);
+        cursor.move_next();
+        // 2 3 4 5 *()
+        assert!(cursor.current_meta().is_none());
+        cursor.move_prev();
+        // 2 3 4 *5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 5);
+        cursor.move_prev();
+        // 2 3 *4 5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 4);
+        cursor.move_prev();
+        // 2 *3 4 5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 3);
+        cursor.move_prev();
+        // *2 3 4 5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 2);
+
+        let frame2 = cursor.take_current().unwrap();
+        // *3 4 5 ()
+        assert_eq!(frame2.meta().mark, 2);
+        assert_eq!(cursor.current_meta().unwrap().mark, 3);
+        cursor.move_next();
+        // 3 *4 5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 4);
+        cursor.move_next();
+        // 3 4 *5 ()
+        assert_eq!(cursor.current_meta().unwrap().mark, 5);
+        // 3 4 *()
+        let frame5 = cursor.take_current().unwrap();
+        assert_eq!(frame5.meta().mark, 5);
+        assert!(cursor.current_meta().is_none());
+
+        assert_eq!(cursor.as_list().size(), 2);
+    }
+}

ostd/src/mm/frame/meta.rs

@@ -63,8 +63,10 @@ use crate::{
 };
 
 /// The maximum number of bytes of the metadata of a frame.
-pub const FRAME_METADATA_MAX_SIZE: usize =
+pub const FRAME_METADATA_MAX_SIZE: usize = META_SLOT_SIZE
-    META_SLOT_SIZE - size_of::<AtomicU64>() - size_of::<FrameMetaVtablePtr>();
+    - size_of::<AtomicU64>()
+    - size_of::<FrameMetaVtablePtr>()
+    - size_of::<AtomicU64>();
 /// The maximum alignment in bytes of the metadata of a frame.
 pub const FRAME_METADATA_MAX_ALIGN: usize = META_SLOT_SIZE;
 
@@ -105,6 +107,15 @@ pub(in crate::mm) struct MetaSlot {
     pub(super) ref_count: AtomicU64,
     /// The virtual table that indicates the type of the metadata.
     pub(super) vtable_ptr: UnsafeCell<MaybeUninit<FrameMetaVtablePtr>>,
+    /// This is only accessed by [`crate::mm::frame::linked_list`].
+    /// It stores 0 if the frame is not in any list, otherwise it stores the
+    /// ID of the list.
+    ///
+    /// It is ugly but allows us to tell if a frame is in a specific list by
+    /// one relaxed read. Otherwise, if we store it conditionally in `storage`
+    /// we would have to ensure that the type is correct before the read, which
+    /// costs a synchronization.
+    pub(super) in_list: AtomicU64,
 }
 
 pub(super) const REF_COUNT_UNUSED: u64 = u64::MAX;
@@ -315,8 +326,8 @@ impl MetaSlot {
     /// The returned pointer should not be dereferenced as mutable unless having
     /// exclusive access to the metadata slot.
     pub(super) unsafe fn dyn_meta_ptr(&self) -> *mut dyn AnyFrameMeta {
-        // SAFETY: The page metadata is valid to be borrowed mutably, since it will never be
+        // SAFETY: The page metadata is valid to be borrowed immutably, since
-        // borrowed immutably after initialization.
+        // it will never be borrowed mutably after initialization.
         let vtable_ptr = unsafe { *self.vtable_ptr.get() };
 
         // SAFETY: The page metadata is initialized and valid.
@@ -498,18 +509,21 @@ fn alloc_meta_frames(tot_nr_frames: usize) -> (usize, Paddr) {
 
     let slots = paddr_to_vaddr(start_paddr) as *mut MetaSlot;
 
-    // Fill the metadata frames with a byte pattern of `REF_COUNT_UNUSED`.
+    // Initialize the metadata slots.
-    debug_assert_eq!(REF_COUNT_UNUSED.to_ne_bytes(), [0xff; 8]);
+    for i in 0..tot_nr_frames {
-    // SAFETY: `slots` and the length is a valid region for the metadata frames
+        // SAFETY: The memory is successfully allocated with `tot_nr_frames`
-    // that are going to be treated as metadata slots. The byte pattern is
+        // slots so the index must be within the range.
-    // valid as the initial value of the reference count (other fields are
+        let slot = unsafe { slots.add(i) };
-    // either not accessed or `MaybeUninit`).
+        // SAFETY: The memory is just allocated so we have exclusive access and
+        // it's valid for writing.
         unsafe {
-        core::ptr::write_bytes(
+            slot.write(MetaSlot {
-            slots as *mut u8,
+                storage: UnsafeCell::new([0; FRAME_METADATA_MAX_SIZE]),
-            0xff,
+                ref_count: AtomicU64::new(REF_COUNT_UNUSED),
-            tot_nr_frames * size_of::<MetaSlot>(),
+                vtable_ptr: UnsafeCell::new(MaybeUninit::uninit()),
-        );
+                in_list: AtomicU64::new(0),
+            })
+        };
     }
 
     (nr_meta_pages, start_paddr)

ostd/src/mm/frame/mod.rs

@@ -32,6 +32,7 @@
 //! can create custom metadata types by implementing the [`AnyFrameMeta`] trait.
 
 pub mod allocator;
+pub mod linked_list;
 pub mod meta;
 pub mod segment;
 pub mod unique;
@@ -162,9 +163,8 @@ impl<M: AnyFrameMeta + ?Sized> Frame<M> {
     /// restored using [`Frame::from_raw`] later. This is useful when some architectural
     /// data structures need to hold the frame handle such as the page table.
     pub(in crate::mm) fn into_raw(self) -> Paddr {
-        let paddr = self.start_paddr();
+        let this = ManuallyDrop::new(self);
-        let _ = ManuallyDrop::new(self);
+        this.start_paddr()
-        paddr
     }
 
     /// Restores a forgotten [`Frame`] from a physical address.

ostd/src/mm/frame/unique.rs

@@ -2,13 +2,13 @@
 
 //! The unique frame pointer that is not shared with others.
 
-use core::{marker::PhantomData, sync::atomic::Ordering};
+use core::{marker::PhantomData, mem::ManuallyDrop, sync::atomic::Ordering};
 
 use super::{
     meta::{GetFrameError, REF_COUNT_UNIQUE},
     AnyFrameMeta, Frame, MetaSlot,
 };
-use crate::mm::{Paddr, PagingLevel, PAGE_SIZE};
+use crate::mm::{frame::mapping, Paddr, PagingConsts, PagingLevel, PAGE_SIZE};
 
 /// An owning frame pointer.
 ///
@@ -99,7 +99,29 @@ impl<M: AnyFrameMeta + ?Sized> UniqueFrame<M> {
         unsafe { &mut *self.slot().dyn_meta_ptr() }
     }
 
-    fn slot(&self) -> &MetaSlot {
+    /// Converts this frame into a raw physical address.
+    pub(crate) fn into_raw(self) -> Paddr {
+        let this = ManuallyDrop::new(self);
+        this.start_paddr()
+    }
+
+    /// Restores a raw physical address back into a unique frame.
+    ///
+    /// # Safety
+    ///
+    /// The caller must ensure that the physical address is valid and points to
+    /// a forgotten frame that was previously casted by [`Self::into_raw`].
+    pub(crate) unsafe fn from_raw(paddr: Paddr) -> Self {
+        let vaddr = mapping::frame_to_meta::<PagingConsts>(paddr);
+        let ptr = vaddr as *const MetaSlot;
+
+        Self {
+            ptr,
+            _marker: PhantomData,
+        }
+    }
+
+    pub(super) fn slot(&self) -> &MetaSlot {
         // SAFETY: `ptr` points to a valid `MetaSlot` that will never be
         // mutably borrowed, so taking an immutable reference to it is safe.
         unsafe { &*self.ptr }