From 44d54f4b232fe10ca4b25580a88aa9d8a415ef48 Mon Sep 17 00:00:00 2001 From: Zhang Junyang Date: Sat, 4 Jan 2025 16:06:44 +0800 Subject: [PATCH] Implement `UniqueFrame` and `Frame::from_in_use` --- ostd/src/mm/frame/allocator.rs | 2 +- ostd/src/mm/frame/meta.rs | 306 +++++++++++++++++++++++++++------ ostd/src/mm/frame/mod.rs | 125 ++++---------- ostd/src/mm/frame/unique.rs | 149 ++++++++++++++++ ostd/src/mm/kspace/mod.rs | 2 +- ostd/src/mm/mod.rs | 1 + 6 files changed, 441 insertions(+), 144 deletions(-) create mode 100644 ostd/src/mm/frame/unique.rs diff --git a/ostd/src/mm/frame/allocator.rs b/ostd/src/mm/frame/allocator.rs index e0a4bc030..3276bc5ae 100644 --- a/ostd/src/mm/frame/allocator.rs +++ b/ostd/src/mm/frame/allocator.rs @@ -60,7 +60,7 @@ impl FrameAllocOptions { .alloc(1) .map(|idx| { let paddr = idx * PAGE_SIZE; - Frame::from_unused(paddr, metadata) + Frame::from_unused(paddr, metadata).unwrap() }) .ok_or(Error::NoMemory)?; diff --git a/ostd/src/mm/frame/meta.rs b/ostd/src/mm/frame/meta.rs index aa702066d..f22562dea 100644 --- a/ostd/src/mm/frame/meta.rs +++ b/ostd/src/mm/frame/meta.rs @@ -43,6 +43,7 @@ use core::{ cell::UnsafeCell, fmt::Debug, mem::{size_of, MaybeUninit}, + result::Result, sync::atomic::{AtomicU32, Ordering}, }; @@ -81,20 +82,23 @@ pub(in crate::mm) struct MetaSlot { /// - the subsequent fields can utilize the padding of the /// reference count to save space. /// - /// Don't access this field with a reference to the slot. - _storage: UnsafeCell<[u8; FRAME_METADATA_MAX_SIZE]>, + /// Don't interpret this field as an array of bytes. It is a + /// placeholder for the metadata of a frame. + storage: UnsafeCell<[u8; FRAME_METADATA_MAX_SIZE]>, /// The reference count of the page. /// /// Specifically, the reference count has the following meaning: /// - `REF_COUNT_UNUSED`: The page is not in use. + /// - `REF_COUNT_UNIQUE`: The page is owned by a [`UniqueFrame`]. /// - `0`: The page is being constructed ([`Frame::from_unused`]) /// or destructured ([`drop_last_in_place`]). /// - `1..REF_COUNT_MAX`: The page is in use. - /// - `REF_COUNT_MAX..REF_COUNT_UNUSED`: Illegal values to + /// - `REF_COUNT_MAX..REF_COUNT_UNIQUE`: Illegal values to /// prevent the reference count from overflowing. Otherwise, /// overflowing the reference count will cause soundness issue. /// /// [`Frame::from_unused`]: super::Frame::from_unused + /// [`UniqueFrame`]: super::unique::UniqueFrame // // Other than this field the fields should be `MaybeUninit`. // See initialization in `alloc_meta_frames`. @@ -104,7 +108,8 @@ pub(in crate::mm) struct MetaSlot { } pub(super) const REF_COUNT_UNUSED: u32 = u32::MAX; -const REF_COUNT_MAX: u32 = i32::MAX as u32; +pub(super) const REF_COUNT_UNIQUE: u32 = u32::MAX - 1; +pub(super) const REF_COUNT_MAX: u32 = i32::MAX as u32; type FrameMetaVtablePtr = core::ptr::DynMetadata; @@ -162,7 +167,122 @@ macro_rules! impl_frame_meta_for { pub use impl_frame_meta_for; +/// The error type for getting the frame from a physical address. +#[derive(Debug)] +pub enum GetFrameError { + /// The frame is in use. + InUse, + /// The frame is not in use. + Unused, + /// The frame is being initialized or destructed. + Busy, + /// The frame is private to an owner of [`UniqueFrame`]. + /// + /// [`UniqueFrame`]: super::unique::UniqueFrame + Unique, + /// The provided physical address is out of bound. + OutOfBound, + /// The provided physical address is not aligned. + NotAligned, +} + +/// Gets the reference to a metadata slot. +pub(super) fn get_slot(paddr: Paddr) -> Result<&'static MetaSlot, GetFrameError> { + if paddr % PAGE_SIZE != 0 { + return Err(GetFrameError::NotAligned); + } + if paddr >= super::MAX_PADDR.load(Ordering::Relaxed) as Paddr { + return Err(GetFrameError::OutOfBound); + } + + let vaddr = mapping::frame_to_meta::(paddr); + let ptr = vaddr as *mut MetaSlot; + + // SAFETY: `ptr` points to a valid `MetaSlot` that will never be + // mutably borrowed, so taking an immutable reference to it is safe. + Ok(unsafe { &*ptr }) +} + impl MetaSlot { + /// Initializes the metadata slot of a frame assuming it is unused. + /// + /// If successful, the function returns a pointer to the metadata slot. + /// And the slot is initialized with the given metadata. + /// + /// The resulting reference count held by the returned pointer is + /// [`REF_COUNT_UNIQUE`] if `as_unique_ptr` is `true`, otherwise `1`. + pub(super) fn get_from_unused( + paddr: Paddr, + metadata: M, + as_unique_ptr: bool, + ) -> Result<*const Self, GetFrameError> { + let slot = get_slot(paddr)?; + + // `Acquire` pairs with the `Release` in `drop_last_in_place` and ensures the metadata + // initialization won't be reordered before this memory compare-and-exchange. + slot.ref_count + .compare_exchange(REF_COUNT_UNUSED, 0, Ordering::Acquire, Ordering::Relaxed) + .map_err(|val| match val { + REF_COUNT_UNIQUE => GetFrameError::Unique, + 0 => GetFrameError::Busy, + _ => GetFrameError::InUse, + })?; + + // SAFETY: The slot now has a reference count of `0`, other threads will + // not access the metadata slot so it is safe to have a mutable reference. + unsafe { slot.write_meta(metadata) }; + + if as_unique_ptr { + // No one can create a `Frame` instance directly from the page + // address, so `Relaxed` is fine here. + slot.ref_count.store(REF_COUNT_UNIQUE, Ordering::Relaxed); + } else { + // `Release` is used to ensure that the metadata initialization + // won't be reordered after this memory store. + slot.ref_count.store(1, Ordering::Release); + } + + Ok(slot as *const MetaSlot) + } + + /// Gets another owning pointer to the metadata slot from the given page. + pub(super) fn get_from_in_use(paddr: Paddr) -> Result<*const Self, GetFrameError> { + let slot = get_slot(paddr)?; + + // Try to increase the reference count for an in-use frame. Otherwise fail. + loop { + match slot.ref_count.load(Ordering::Relaxed) { + REF_COUNT_UNUSED => return Err(GetFrameError::Unused), + REF_COUNT_UNIQUE => return Err(GetFrameError::Unique), + 0 => return Err(GetFrameError::Busy), + last_ref_cnt => { + if last_ref_cnt >= REF_COUNT_MAX { + // See `Self::inc_ref_count` for the explanation. + abort(); + } + // Using `Acquire` here to pair with `get_from_unused` or + // ` as From>>::from` (who must be + // performed after writing the metadata). + // + // It ensures that the written metadata will be visible to us. + if slot + .ref_count + .compare_exchange_weak( + last_ref_cnt, + last_ref_cnt + 1, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_ok() + { + return Ok(slot as *const MetaSlot); + } + } + } + core::hint::spin_loop(); + } + } + /// Increases the frame reference count by one. /// /// # Safety @@ -179,60 +299,138 @@ impl MetaSlot { abort(); } } -} -/// An internal routine in dropping implementations. -/// -/// # Safety -/// -/// The caller should ensure that the pointer points to a frame's metadata slot. The -/// frame should have a last handle to the frame, and the frame is about to be dropped, -/// as the metadata slot after this operation becomes uninitialized. -pub(super) unsafe fn drop_last_in_place(ptr: *mut MetaSlot) { - // SAFETY: `ptr` points to a valid `MetaSlot` that will never be mutably borrowed, so taking an - // immutable reference to it is always safe. - let slot = unsafe { &*ptr }; - - // This should be guaranteed as a safety requirement. - debug_assert_eq!(slot.ref_count.load(Ordering::Relaxed), 0); - - let paddr = mapping::meta_to_frame::(ptr as Vaddr); - - // SAFETY: We have exclusive access to the frame metadata. - let vtable_ptr = unsafe { &mut *slot.vtable_ptr.get() }; - // SAFETY: The frame metadata is initialized and valid. - let vtable_ptr = unsafe { vtable_ptr.assume_init_read() }; - - let meta_ptr: *mut dyn AnyFrameMeta = core::ptr::from_raw_parts_mut(ptr, vtable_ptr); - - // SAFETY: The implementer of the frame metadata decides that if the frame - // is safe to be read or not. - let mut reader = - unsafe { VmReader::from_kernel_space(paddr_to_vaddr(paddr) as *const u8, PAGE_SIZE) }; - - // SAFETY: `ptr` points to the metadata storage which is valid to be mutably borrowed under - // `vtable_ptr` because the metadata is valid, the vtable is correct, and we have the exclusive - // access to the frame metadata. - unsafe { - // Invoke the custom `on_drop` handler. - (*meta_ptr).on_drop(&mut reader); - // Drop the frame metadata. - core::ptr::drop_in_place(meta_ptr); + /// Gets the corresponding frame's physical address. + pub(super) fn frame_paddr(&self) -> Paddr { + mapping::meta_to_frame::(self as *const MetaSlot as Vaddr) } - // `Release` pairs with the `Acquire` in `Frame::from_unused` and ensures `drop_in_place` won't - // be reordered after this memory store. - slot.ref_count.store(REF_COUNT_UNUSED, Ordering::Release); + /// Gets a dynamically typed pointer to the stored metadata. + /// + /// # Safety + /// + /// The caller should ensure that: + /// - the stored metadata is initialized (by [`Self::write_meta`]) and valid. + /// + /// The returned pointer should not be dereferenced as mutable unless having + /// exclusive access to the metadata slot. + pub(super) unsafe fn dyn_meta_ptr(&self) -> *mut dyn AnyFrameMeta { + // SAFETY: The page metadata is valid to be borrowed mutably, since it will never be + // borrowed immutably after initialization. + let vtable_ptr = unsafe { *self.vtable_ptr.get() }; - // Deallocate the frame. - // It would return the frame to the allocator for further use. This would be done - // after the release of the metadata to avoid re-allocation before the metadata - // is reset. - allocator::FRAME_ALLOCATOR - .get() - .unwrap() - .lock() - .dealloc(paddr / PAGE_SIZE, 1); + // SAFETY: The page metadata is initialized and valid. + let vtable_ptr = *unsafe { vtable_ptr.assume_init_ref() }; + + let meta_ptr: *mut dyn AnyFrameMeta = + core::ptr::from_raw_parts_mut(self as *const MetaSlot as *mut MetaSlot, vtable_ptr); + + meta_ptr + } + + /// Gets the stored metadata as type `M`. + /// + /// Calling the method should be safe, but using the returned pointer would + /// be unsafe. Specifically, the derefernecer should ensure that: + /// - the stored metadata is initialized (by [`Self::write_meta`]) and + /// valid; + /// - the initialized metadata is of type `M`; + /// - the returned pointer should not be dereferenced as mutable unless + /// having exclusive access to the metadata slot. + pub(super) fn as_meta_ptr(&self) -> *mut M { + self.storage.get() as *mut M + } + + /// Writes the metadata to the slot without reading or dropping the previous value. + /// + /// # Safety + /// + /// The caller should have exclusive access to the metadata slot's fields. + pub(super) unsafe fn write_meta(&self, metadata: M) { + // Checking unsafe preconditions of the `AnyFrameMeta` trait. + // We can't debug assert until we fix the constant generic bonds in + // the linked list meta. + assert!(size_of::() <= FRAME_METADATA_MAX_SIZE); + assert!(align_of::() <= FRAME_METADATA_MAX_ALIGN); + + // SAFETY: Caller ensures that the access to the fields are exclusive. + let vtable_ptr = unsafe { &mut *self.vtable_ptr.get() }; + vtable_ptr.write(core::ptr::metadata(&metadata as &dyn AnyFrameMeta)); + + let ptr = self.storage.get(); + // SAFETY: + // 1. `ptr` points to the metadata storage. + // 2. The size and the alignment of the metadata storage is large enough to hold `M` + // (guaranteed by the safety requirement of the `AnyFrameMeta` trait). + // 3. We have exclusive access to the metadata storage (guaranteed by the caller). + unsafe { ptr.cast::().write(metadata) }; + } + + /// Drops the metadata and deallocates the frame. + /// + /// # Safety + /// + /// The caller should ensure that: + /// - the reference count is `0` (so we are the sole owner of the frame); + /// - the metadata is initialized; + pub(super) unsafe fn drop_last_in_place(&self) { + // This should be guaranteed as a safety requirement. + debug_assert_eq!(self.ref_count.load(Ordering::Relaxed), 0); + + // SAFETY: The caller ensures safety. + unsafe { self.drop_meta_in_place() }; + + // `Release` pairs with the `Acquire` in `Frame::from_unused` and ensures + // `drop_meta_in_place` won't be reordered after this memory store. + self.ref_count.store(REF_COUNT_UNUSED, Ordering::Release); + + // Deallocate the frame. + // It would return the frame to the allocator for further use. This would be done + // after the release of the metadata to avoid re-allocation before the metadata + // is reset. + allocator::FRAME_ALLOCATOR + .get() + .unwrap() + .lock() + .dealloc(self.frame_paddr() / PAGE_SIZE, 1); + } + + /// Drops the metadata of a slot in place. + /// + /// After this operation, the metadata becomes uninitialized. Any access to the + /// metadata is undefined behavior unless it is re-initialized by [`Self::write_meta`]. + /// + /// # Safety + /// + /// The caller should ensure that: + /// - the reference count is `0` (so we are the sole owner of the frame); + /// - the metadata is initialized; + pub(super) unsafe fn drop_meta_in_place(&self) { + let paddr = self.frame_paddr(); + + // SAFETY: We have exclusive access to the frame metadata. + let vtable_ptr = unsafe { &mut *self.vtable_ptr.get() }; + // SAFETY: The frame metadata is initialized and valid. + let vtable_ptr = unsafe { vtable_ptr.assume_init_read() }; + + let meta_ptr: *mut dyn AnyFrameMeta = + core::ptr::from_raw_parts_mut(self.storage.get(), vtable_ptr); + + // SAFETY: The implementer of the frame metadata decides that if the frame + // is safe to be read or not. + let mut reader = + unsafe { VmReader::from_kernel_space(paddr_to_vaddr(paddr) as *const u8, PAGE_SIZE) }; + + // SAFETY: `ptr` points to the metadata storage which is valid to be mutably borrowed under + // `vtable_ptr` because the metadata is valid, the vtable is correct, and we have the exclusive + // access to the frame metadata. + unsafe { + // Invoke the custom `on_drop` handler. + (*meta_ptr).on_drop(&mut reader); + // Drop the frame metadata. + core::ptr::drop_in_place(meta_ptr); + } + } } /// The metadata of frames that holds metadata of frames. diff --git a/ostd/src/mm/frame/mod.rs b/ostd/src/mm/frame/mod.rs index cf8a88fe0..163e63ce6 100644 --- a/ostd/src/mm/frame/mod.rs +++ b/ostd/src/mm/frame/mod.rs @@ -34,17 +34,16 @@ pub mod allocator; pub mod meta; pub mod segment; +pub mod unique; pub mod untyped; use core::{ marker::PhantomData, - sync::atomic::{AtomicU32, AtomicUsize, Ordering}, + mem::ManuallyDrop, + sync::atomic::{AtomicUsize, Ordering}, }; -use meta::{ - mapping, AnyFrameMeta, MetaSlot, FRAME_METADATA_MAX_ALIGN, FRAME_METADATA_MAX_SIZE, - REF_COUNT_UNUSED, -}; +use meta::{mapping, AnyFrameMeta, GetFrameError, MetaSlot, REF_COUNT_UNUSED}; pub use segment::Segment; use untyped::{AnyUFrameMeta, UFrame}; @@ -64,10 +63,6 @@ static MAX_PADDR: AtomicUsize = AtomicUsize::new(0); #[derive(Debug)] #[repr(transparent)] pub struct Frame { - // TODO: We may use a `NonNull` here to make the frame a maybe-fat - // pointer and implement `CoerceUnsized` to avoid `From`s. However this is - // not quite feasible currently because we cannot cast a must-be-fat - // pointer (`*const dyn AnyFrameMeta`) to a maybe-fat pointer (`NonNull`). ptr: *const MetaSlot, _marker: PhantomData, } @@ -81,70 +76,41 @@ impl Frame { /// /// The caller should provide the initial metadata of the page. /// - /// # Panics - /// - /// The function panics if: - /// - the physical address is out of bound or not aligned; - /// - the page is already in use. - pub fn from_unused(paddr: Paddr, metadata: M) -> Self { - assert!(paddr % PAGE_SIZE == 0); - assert!(paddr < MAX_PADDR.load(Ordering::Relaxed) as Paddr); - - // Checking unsafe preconditions of the `AnyFrameMeta` trait. - debug_assert!(size_of::() <= FRAME_METADATA_MAX_SIZE); - debug_assert!(align_of::() <= FRAME_METADATA_MAX_ALIGN); - - let vaddr = mapping::frame_to_meta::(paddr); - let ptr = vaddr as *const MetaSlot; - - // SAFETY: `ptr` points to a valid `MetaSlot` that will never be mutably borrowed, so taking an - // immutable reference to it is always safe. - let slot = unsafe { &*ptr }; - - // `Acquire` pairs with the `Release` in `drop_last_in_place` and ensures the metadata - // initialization won't be reordered before this memory compare-and-exchange. - slot.ref_count - .compare_exchange(REF_COUNT_UNUSED, 0, Ordering::Acquire, Ordering::Relaxed) - .expect("Frame already in use when trying to get a new handle"); - - // SAFETY: We have exclusive access to the page metadata. These fields are mutably - // borrowed only once. - let vtable_ptr = unsafe { &mut *slot.vtable_ptr.get() }; - vtable_ptr.write(core::ptr::metadata(&metadata as &dyn AnyFrameMeta)); - - // SAFETY: - // 1. `ptr` points to the first field of `MetaSlot` (guaranteed by `repr(C)`), which is the - // metadata storage. - // 2. The size and the alignment of the metadata storage is large enough to hold `M` - // (guaranteed by the safety requirement of the `AnyFrameMeta` trait). - // 3. We have exclusive access to the metadata storage (guaranteed by the reference count). - unsafe { ptr.cast::().cast_mut().write(metadata) }; - - // Assuming no one can create a `Frame` instance directly from the page address, `Relaxed` - // is fine here. Otherwise, we should use `Release` to ensure that the metadata - // initialization won't be reordered after this memory store. - slot.ref_count.store(1, Ordering::Relaxed); - - Self { - ptr, + /// If the provided frame is not truly unused at the moment, it will return + /// an error. If wanting to acquire a frame that is already in use, use + /// [`Frame::from_in_use`] instead. + pub fn from_unused(paddr: Paddr, metadata: M) -> Result { + Ok(Self { + ptr: MetaSlot::get_from_unused(paddr, metadata, false)?, _marker: PhantomData, - } + }) } /// Gets the metadata of this page. pub fn meta(&self) -> &M { - // SAFETY: `self.ptr` points to the metadata storage which is valid to - // be immutably borrowed as `M` because the type is correct, it lives - // under the given lifetime, and no one will mutably borrow the page - // metadata after initialization. - unsafe { &*self.ptr.cast() } + // SAFETY: The type is tracked by the type system. + unsafe { &*self.slot().as_meta_ptr::() } + } +} + +impl Frame { + /// Gets a dynamically typed [`Frame`] from a raw, in-use page. + /// + /// If the provided frame is not in use at the moment, it will return an error. + /// + /// The returned frame will have an extra reference count to the frame. + pub fn from_in_use(paddr: Paddr) -> Result { + Ok(Self { + ptr: MetaSlot::get_from_in_use(paddr)?, + _marker: PhantomData, + }) } } impl Frame { /// Gets the physical address of the start of the frame. pub fn start_paddr(&self) -> Paddr { - mapping::meta_to_frame::(self.ptr as Vaddr) + self.slot().frame_paddr() } /// Gets the paging level of this page. @@ -167,21 +133,8 @@ impl Frame { /// /// If the type is known at compile time, use [`Frame::meta`] instead. pub fn dyn_meta(&self) -> &dyn AnyFrameMeta { - let slot = self.slot(); - - // SAFETY: The page metadata is valid to be borrowed immutably, since it will never be - // borrowed mutably after initialization. - let vtable_ptr = unsafe { &*slot.vtable_ptr.get() }; - - // SAFETY: The page metadata is initialized and valid. - let vtable_ptr = *unsafe { vtable_ptr.assume_init_ref() }; - - let meta_ptr: *const dyn AnyFrameMeta = core::ptr::from_raw_parts(self.ptr, vtable_ptr); - - // SAFETY: `self.ptr` points to the metadata storage which is valid to be immutably - // borrowed under `vtable_ptr` because the vtable is correct, it lives under the given - // lifetime, and no one will mutably borrow the page metadata after initialization. - unsafe { &*meta_ptr } + // SAFETY: The metadata is initialized and valid. + unsafe { &*self.slot().dyn_meta_ptr() } } /// Gets the reference count of the frame. @@ -196,11 +149,9 @@ impl Frame { /// reference count can be changed by other threads at any time including /// potentially between calling this method and acting on the result. pub fn reference_count(&self) -> u32 { - self.ref_count().load(Ordering::Relaxed) - } - - fn ref_count(&self) -> &AtomicU32 { - unsafe { &(*self.ptr).ref_count } + let refcnt = self.slot().ref_count.load(Ordering::Relaxed); + debug_assert!(refcnt < meta::REF_COUNT_MAX); + refcnt } /// Forgets the handle to the frame. @@ -212,11 +163,11 @@ impl Frame { /// data structures need to hold the frame handle such as the page table. pub(in crate::mm) fn into_raw(self) -> Paddr { let paddr = self.start_paddr(); - core::mem::forget(self); + let _ = ManuallyDrop::new(self); paddr } - /// Restores a forgotten `Frame` from a physical address. + /// Restores a forgotten [`Frame`] from a physical address. /// /// # Safety /// @@ -224,7 +175,7 @@ impl Frame { /// [`Frame::into_raw`]. /// /// And the restoring operation should only be done once for a forgotten - /// `Frame`. Otherwise double-free will happen. + /// [`Frame`]. Otherwise double-free will happen. /// /// Also, the caller ensures that the usage of the frame is correct. There's /// no checking of the usage in this function. @@ -268,9 +219,7 @@ impl Drop for Frame { core::sync::atomic::fence(Ordering::Acquire); // SAFETY: this is the last reference and is about to be dropped. - unsafe { - meta::drop_last_in_place(self.ptr as *mut MetaSlot); - } + unsafe { self.slot().drop_last_in_place() }; } } } diff --git a/ostd/src/mm/frame/unique.rs b/ostd/src/mm/frame/unique.rs new file mode 100644 index 000000000..0fe785d79 --- /dev/null +++ b/ostd/src/mm/frame/unique.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! The unique frame pointer that is not shared with others. + +use core::{marker::PhantomData, sync::atomic::Ordering}; + +use super::{ + meta::{GetFrameError, REF_COUNT_UNIQUE}, + AnyFrameMeta, Frame, MetaSlot, +}; +use crate::mm::{Paddr, PagingLevel, PAGE_SIZE}; + +/// An owning frame pointer. +/// +/// Unlike [`Frame`], the frame pointed to by this pointer is not shared with +/// others. So a mutable reference to the metadata is available for the frame. +#[derive(Debug)] +#[repr(transparent)] +pub struct UniqueFrame { + ptr: *const MetaSlot, + _marker: PhantomData, +} + +unsafe impl Send for UniqueFrame {} + +unsafe impl Sync for UniqueFrame {} + +impl UniqueFrame { + /// Gets a [`UniqueFrame`] with a specific usage from a raw, unused page. + /// + /// The caller should provide the initial metadata of the page. + pub fn from_unused(paddr: Paddr, metadata: M) -> Result { + Ok(Self { + ptr: MetaSlot::get_from_unused(paddr, metadata, true)?, + _marker: PhantomData, + }) + } + + /// Repurposes the frame with a new metadata. + pub fn repurpose(self, metadata: M1) -> UniqueFrame { + // SAFETY: We are the sole owner and the metadata is initialized. + unsafe { self.slot().drop_meta_in_place() }; + // SAFETY: We are the sole owner. + unsafe { self.slot().write_meta(metadata) }; + // SAFETY: The metadata is initialized with type `M1`. + unsafe { core::mem::transmute(self) } + } + + /// Gets the metadata of this page. + pub fn meta(&self) -> &M { + // SAFETY: The type is tracked by the type system. + unsafe { &*self.slot().as_meta_ptr::() } + } + + /// Gets the mutable metadata of this page. + pub fn meta_mut(&mut self) -> &mut M { + // SAFETY: The type is tracked by the type system. + // And we have the exclusive access to the metadata. + unsafe { &mut *self.slot().as_meta_ptr::() } + } +} + +impl UniqueFrame { + /// Gets the physical address of the start of the frame. + pub fn start_paddr(&self) -> Paddr { + self.slot().frame_paddr() + } + + /// Gets the paging level of this page. + /// + /// This is the level of the page table entry that maps the frame, + /// which determines the size of the frame. + /// + /// Currently, the level is always 1, which means the frame is a regular + /// page frame. + pub const fn level(&self) -> PagingLevel { + 1 + } + + /// Gets the size of this page in bytes. + pub const fn size(&self) -> usize { + PAGE_SIZE + } + + /// Gets the dyncamically-typed metadata of this frame. + /// + /// If the type is known at compile time, use [`Frame::meta`] instead. + pub fn dyn_meta(&self) -> &dyn AnyFrameMeta { + // SAFETY: The metadata is initialized and valid. + unsafe { &*self.slot().dyn_meta_ptr() } + } + + /// Gets the dyncamically-typed metadata of this frame. + /// + /// If the type is known at compile time, use [`Frame::meta`] instead. + pub fn dyn_meta_mut(&mut self) -> &mut dyn AnyFrameMeta { + // SAFETY: The metadata is initialized and valid. We have the exclusive + // access to the frame. + unsafe { &mut *self.slot().dyn_meta_ptr() } + } + + fn slot(&self) -> &MetaSlot { + // SAFETY: `ptr` points to a valid `MetaSlot` that will never be + // mutably borrowed, so taking an immutable reference to it is safe. + unsafe { &*self.ptr } + } +} + +impl Drop for UniqueFrame { + fn drop(&mut self) { + self.slot().ref_count.store(0, Ordering::Relaxed); + // SAFETY: We are the sole owner and the reference count is 0. + // The slot is initialized. + unsafe { self.slot().drop_last_in_place() }; + } +} + +impl From> for Frame { + fn from(unique: UniqueFrame) -> Self { + // The `Release` ordering make sure that previous writes are visible + // before the reference count is set to 1. It pairs with + // `MetaSlot::get_from_in_use`. + unique.slot().ref_count.store(1, Ordering::Release); + // SAFETY: The internal representation is now the same. + unsafe { core::mem::transmute(unique) } + } +} + +impl TryFrom> for UniqueFrame { + type Error = Frame; + + /// Tries to get a unique frame from a shared frame. + /// + /// If the reference count is not 1, the frame is returned back. + fn try_from(frame: Frame) -> Result { + match frame.slot().ref_count.compare_exchange( + 1, + REF_COUNT_UNIQUE, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + // SAFETY: The reference count is now `REF_COUNT_UNIQUE`. + Ok(unsafe { core::mem::transmute::, UniqueFrame>(frame) }) + } + Err(_) => Err(frame), + } + } +} diff --git a/ostd/src/mm/kspace/mod.rs b/ostd/src/mm/kspace/mod.rs index 2c9843e83..c14d82b08 100644 --- a/ostd/src/mm/kspace/mod.rs +++ b/ostd/src/mm/kspace/mod.rs @@ -214,7 +214,7 @@ pub fn init_kernel_page_table(meta_pages: Segment) { }; let mut cursor = kpt.cursor_mut(&from).unwrap(); for frame_paddr in to.step_by(PAGE_SIZE) { - let page = Frame::::from_unused(frame_paddr, KernelMeta); + let page = Frame::::from_unused(frame_paddr, KernelMeta).unwrap(); // SAFETY: we are doing mappings for the kernel. unsafe { let _old = cursor.map(page.into(), prop); diff --git a/ostd/src/mm/mod.rs b/ostd/src/mm/mod.rs index d1353d05b..1fa0de3b2 100644 --- a/ostd/src/mm/mod.rs +++ b/ostd/src/mm/mod.rs @@ -27,6 +27,7 @@ pub use self::{ frame::{ allocator::FrameAllocOptions, segment::{Segment, USegment}, + unique::UniqueFrame, untyped::{AnyUFrameMeta, UFrame, UntypedMem}, Frame, },