diff --git a/kernel/src/vm/vmo/dyn_cap.rs b/kernel/src/vm/vmo/dyn_cap.rs index 3a4f0619d..7596c442b 100644 --- a/kernel/src/vm/vmo/dyn_cap.rs +++ b/kernel/src/vm/vmo/dyn_cap.rs @@ -5,45 +5,57 @@ use core::ops::Range; use aster_rights::{Rights, TRights}; use ostd::mm::{UFrame, VmIo}; -use super::{CommitFlags, Vmo, VmoRightsOp}; +use super::{CommitFlags, Vmo, VmoCommitError, VmoRightsOp}; use crate::prelude::*; impl Vmo { - /// Commits a page at specific offset - pub fn commit_page(&self, offset: usize) -> Result { - self.check_rights(Rights::WRITE)?; - self.0.commit_page(offset) - } - - /// Commits the pages specified in the range (in bytes). + /// Commits a page at specific offset. /// - /// The range must be within the size of the VMO. - /// - /// The start and end addresses will be rounded down and up to page boundaries. + /// If the commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. /// /// # Access rights /// /// The method requires the Write right. - pub fn commit(&self, range: Range) -> Result<()> { + pub fn try_commit_page(&self, offset: usize) -> core::result::Result { self.check_rights(Rights::WRITE)?; - self.0.operate_on_range( - &range, - |commit_fn| commit_fn().map(|_| ()), - CommitFlags::empty(), - )?; - Ok(()) + self.0.try_commit_page(offset) + } + + /// Commits a page at a specific page index. + /// + /// This method may involve I/O operations if the VMO needs to fetch + /// a page from the underlying page cache. + /// + /// # Access rights + /// + /// The method requires the Write right. + pub fn commit_on(&self, page_idx: usize, commit_flags: CommitFlags) -> Result { + self.check_rights(Rights::WRITE)?; + self.0.commit_on(page_idx, commit_flags) } /// Traverses the indices within a specified range of a VMO sequentially. + /// /// For each index position, you have the option to commit the page as well as /// perform other operations. - pub(in crate::vm) fn operate_on_range(&self, range: &Range, operate: F) -> Result<()> + /// + /// Once a commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. + /// + /// # Access rights + /// + /// The method requires the Write right. + pub(in crate::vm) fn try_operate_on_range( + &self, + range: &Range, + operate: F, + ) -> core::result::Result<(), VmoCommitError> where - F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + F: FnMut( + &mut dyn FnMut() -> core::result::Result, + ) -> core::result::Result<(), VmoCommitError>, { self.check_rights(Rights::WRITE)?; - self.0 - .operate_on_range(range, operate, CommitFlags::empty()) + self.0.try_operate_on_range(range, operate) } /// Decommits the pages specified in the range (in bytes). @@ -94,19 +106,6 @@ impl Vmo { Ok(Self(self.0.clone(), self.1)) } - /// Creates a new VMO that replicates the original capability, initially representing - /// the same physical pages. - /// Changes to the permissions and commits/replacements of internal pages in the original VMO - /// and the new VMO will not affect each other. - /// - /// # Access rights - /// - /// The method requires the Dup right. - pub fn dup_independent(&self) -> Result { - self.check_rights(Rights::DUP | Rights::WRITE)?; - Ok(Vmo(Arc::new(super::Vmo_::clone(&self.0)), self.1)) - } - /// Replaces the page at the `page_idx` in the VMO with the input `page`. /// /// # Access rights diff --git a/kernel/src/vm/vmo/mod.rs b/kernel/src/vm/vmo/mod.rs index 77c94dc0b..7d14ad166 100644 --- a/kernel/src/vm/vmo/mod.rs +++ b/kernel/src/vm/vmo/mod.rs @@ -5,14 +5,18 @@ //! Virtual Memory Objects (VMOs). -use core::ops::Range; +use core::{ + ops::Range, + sync::atomic::{AtomicUsize, Ordering}, +}; use align_ext::AlignExt; use aster_rights::Rights; use ostd::{ - collections::xarray::{CursorMut, XArray}, mm::{FrameAllocOptions, UFrame, UntypedMem, VmReader, VmWriter}, + task::disable_preempt, }; +use xarray::{Cursor, LockedXArray, XArray}; use crate::prelude::*; @@ -125,55 +129,47 @@ bitflags! { } } -/// `Pages` is the struct that manages the `UFrame`s stored in `Vmo_`. -pub(super) enum Pages { - /// `Pages` that cannot be resized. This kind of `Pages` will have a constant size. - Nonresizable(Mutex>, usize), - /// `Pages` that can be resized and have a variable size. - Resizable(Mutex<(XArray, usize)>), +/// The error type used for commit operations of [`Vmo`]. +#[derive(Debug)] +pub enum VmoCommitError { + /// Represents a general error raised during the commit operation. + Err(Error), + /// Represents that the commit operation need to do I/O operation on the + /// wrapped index. + NeedIo(usize), } -impl Clone for Pages { - fn clone(&self) -> Self { - match self { - Self::Nonresizable(_, _) => { - self.with(|pages, size| Self::Nonresizable(Mutex::new(pages.clone()), size)) - } - Self::Resizable(_) => { - self.with(|pages, size| Self::Resizable(Mutex::new((pages.clone(), size)))) - } - } +impl From for VmoCommitError { + fn from(e: Error) -> Self { + VmoCommitError::Err(e) } } -impl Pages { - fn with(&self, func: F) -> R - where - F: FnOnce(&mut XArray, usize) -> R, - { - match self { - Self::Nonresizable(pages, size) => func(&mut pages.lock(), *size), - Self::Resizable(pages) => { - let mut lock = pages.lock(); - let size = lock.1; - func(&mut lock.0, size) - } - } +impl From for VmoCommitError { + fn from(e: ostd::Error) -> Self { + Error::from(e).into() } } /// `Vmo_` is the structure that actually manages the content of VMO. +/// /// Broadly speaking, there are two types of VMO: -/// 1. File-backed VMO: the VMO backed by a file and resides in the `PageCache`, -/// which includes a pager to provide it with actual pages. -/// 2. Anonymous VMO: the VMO without a file backup, which does not have a pager. -#[derive(Clone)] +/// 1. File-backed VMO: the VMO backed by a file and resides in the page cache, +/// which includes a [`Pager`] to provide it with actual pages. +/// 2. Anonymous VMO: the VMO without a file backup, which does not have a `Pager`. pub(super) struct Vmo_ { pager: Option>, /// Flags flags: VmoFlags, /// The virtual pages where the VMO resides. - pages: Pages, + pages: XArray, + /// The size of the VMO. + /// + /// Note: This size may not necessarily match the size of the `pages`, but it is + /// required here that modifications to the size can only be made after locking + /// the [`XArray`] in the `pages` field. Therefore, the size read after locking the + /// `pages` will be the latest size. + size: AtomicUsize, } impl Debug for Vmo_ { @@ -202,111 +198,155 @@ impl CommitFlags { impl Vmo_ { /// Prepares a new `UFrame` for the target index in pages, returns this new frame. - fn prepare_page(&self, page_idx: usize) -> Result { + /// + /// This operation may involve I/O operations if the VMO is backed by a pager. + fn prepare_page(&self, page_idx: usize, commit_flags: CommitFlags) -> Result { match &self.pager { None => Ok(FrameAllocOptions::new().alloc_frame()?.into()), - Some(pager) => pager.commit_page(page_idx), - } - } - - /// Prepares a new `UFrame` for the target index in the VMO, returns this new frame. - fn prepare_overwrite(&self, page_idx: usize) -> Result { - if let Some(pager) = &self.pager { - pager.commit_overwrite(page_idx) - } else { - Ok(FrameAllocOptions::new().alloc_frame()?.into()) - } - } - - fn commit_with_cursor( - &self, - cursor: &mut CursorMut<'_, UFrame>, - commit_flags: CommitFlags, - ) -> Result { - let new_page = { - if let Some(committed_page) = cursor.load() { - // Fast path: return the page directly. - return Ok(committed_page.clone()); - } else if commit_flags.will_overwrite() { - // In this case, the page will be completely overwritten. - self.prepare_overwrite(cursor.index() as usize)? - } else { - self.prepare_page(cursor.index() as usize)? + Some(pager) => { + if commit_flags.will_overwrite() { + pager.commit_overwrite(page_idx) + } else { + pager.commit_page(page_idx) + } } - }; + } + } + + /// Commits a page at a specific page index. + /// + /// This method may involve I/O operations if the VMO needs to fetch a page from + /// the underlying page cache. + pub fn commit_on(&self, page_idx: usize, commit_flags: CommitFlags) -> Result { + let new_page = self.prepare_page(page_idx, commit_flags)?; + + let mut locked_pages = self.pages.lock(); + if page_idx * PAGE_SIZE > self.size() { + return_errno_with_message!(Errno::EINVAL, "the offset is outside the VMO"); + } + + let mut cursor = locked_pages.cursor_mut(page_idx as u64); + if let Some(page) = cursor.load() { + return Ok(page.clone()); + } cursor.store(new_page.clone()); Ok(new_page) } - /// Commits the page corresponding to the target offset in the VMO and return that page. - /// If the current offset has already been committed, the page will be returned directly. - pub fn commit_page(&self, offset: usize) -> Result { - let page_idx = offset / PAGE_SIZE; - self.pages.with(|pages, size| { - if offset >= size { - return_errno_with_message!(Errno::EINVAL, "the offset is outside the VMO"); - } - let mut cursor = pages.cursor_mut(page_idx as u64); - self.commit_with_cursor(&mut cursor, CommitFlags::empty()) - }) + fn try_commit_with_cursor( + &self, + cursor: &mut Cursor<'_, UFrame>, + ) -> core::result::Result { + if let Some(committed_page) = cursor.load() { + return Ok(committed_page.clone()); + } + + if let Some(pager) = &self.pager { + // FIXME: Here `Vmo` treat all instructions in `pager` as I/O instructions + // since it needs to take the inner `Mutex` lock and users also cannot hold a + // `SpinLock` to do such instructions. This workaround may introduce some performance + // issues. In the future we should solve the redundancy of `Vmo` and the pagecache + // make sure return such error when really needing I/Os. + return Err(VmoCommitError::NeedIo(cursor.index() as usize)); + } + + let frame = self.commit_on(cursor.index() as usize, CommitFlags::empty())?; + Ok(frame) } - /// Decommits the page corresponding to the target offset in the VMO. - fn decommit_page(&mut self, offset: usize) -> Result<()> { + /// Commits the page corresponding to the target offset in the VMO. + /// + /// If the commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. + pub fn try_commit_page(&self, offset: usize) -> core::result::Result { let page_idx = offset / PAGE_SIZE; - self.pages.with(|pages, size| { - if offset >= size { - return_errno_with_message!(Errno::EINVAL, "the offset is outside the VMO"); - } - let mut cursor = pages.cursor_mut(page_idx as u64); - if cursor.remove().is_some() - && let Some(pager) = &self.pager - { - pager.decommit_page(page_idx)?; - } - Ok(()) - }) + if offset >= self.size() { + return Err(VmoCommitError::Err(Error::with_message( + Errno::EINVAL, + "the offset is outside the VMO", + ))); + } + + let guard = disable_preempt(); + let mut cursor = self.pages.cursor(&guard, page_idx as u64); + self.try_commit_with_cursor(&mut cursor) } /// Traverses the indices within a specified range of a VMO sequentially. + /// /// For each index position, you have the option to commit the page as well as /// perform other operations. - pub fn operate_on_range( + /// + /// Once a commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. + pub fn try_operate_on_range( &self, range: &Range, mut operate: F, + ) -> core::result::Result<(), VmoCommitError> + where + F: FnMut( + &mut dyn FnMut() -> core::result::Result, + ) -> core::result::Result<(), VmoCommitError>, + { + if range.end > self.size() { + return Err(VmoCommitError::Err(Error::with_message( + Errno::EINVAL, + "operated range exceeds the vmo size", + ))); + } + + let page_idx_range = get_page_idx_range(range); + let guard = disable_preempt(); + let mut cursor = self.pages.cursor(&guard, page_idx_range.start as u64); + for page_idx in page_idx_range { + let mut commit_fn = || self.try_commit_with_cursor(&mut cursor); + operate(&mut commit_fn)?; + cursor.next(); + } + Ok(()) + } + + /// Traverses the indices within a specified range of a VMO sequentially. + /// + /// For each index position, you have the option to commit the page as well as + /// perform other operations. + /// + /// This method may involve I/O operations if the VMO needs to fetch a page from + /// the underlying page cache. + fn operate_on_range( + &self, + mut range: Range, + mut operate: F, commit_flags: CommitFlags, ) -> Result<()> where - F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + F: FnMut( + &mut dyn FnMut() -> core::result::Result, + ) -> core::result::Result<(), VmoCommitError>, { - self.pages.with(|pages, size| { - if range.end > size { - return_errno_with_message!(Errno::EINVAL, "operated range exceeds the vmo size"); + 'retry: loop { + let res = self.try_operate_on_range(&range, &mut operate); + match res { + Ok(_) => return Ok(()), + Err(VmoCommitError::Err(e)) => return Err(e), + Err(VmoCommitError::NeedIo(index)) => { + self.commit_on(index, commit_flags)?; + range.start = index * PAGE_SIZE; + continue 'retry; + } } - - let page_idx_range = get_page_idx_range(range); - let mut cursor = pages.cursor_mut(page_idx_range.start as u64); - for page_idx in page_idx_range { - let mut commit_fn = || self.commit_with_cursor(&mut cursor, commit_flags); - operate(&mut commit_fn)?; - cursor.next(); - } - Ok(()) - }) + } } /// Decommits a range of pages in the VMO. pub fn decommit(&self, range: Range) -> Result<()> { - self.pages.with(|pages, size| { - if range.end > size { - return_errno_with_message!(Errno::EINVAL, "operated range exceeds the vmo size"); - } + let locked_pages = self.pages.lock(); + if range.end > self.size() { + return_errno_with_message!(Errno::EINVAL, "operated range exceeds the vmo size"); + } - self.decommit_pages(pages, range)?; - Ok(()) - }) + self.decommit_pages(locked_pages, range)?; + Ok(()) } /// Reads the specified amount of buffer content starting from the target offset in the VMO. @@ -315,14 +355,19 @@ impl Vmo_ { let read_range = offset..(offset + read_len); let mut read_offset = offset % PAGE_SIZE; - let read = move |commit_fn: &mut dyn FnMut() -> Result| { - let frame = commit_fn()?; - frame.reader().skip(read_offset).read_fallible(writer)?; - read_offset = 0; - Ok(()) - }; + let read = + move |commit_fn: &mut dyn FnMut() -> core::result::Result| { + let frame = commit_fn()?; + frame + .reader() + .skip(read_offset) + .read_fallible(writer) + .map_err(|e| VmoCommitError::from(e.0))?; + read_offset = 0; + Ok(()) + }; - self.operate_on_range(&read_range, read, CommitFlags::empty()) + self.operate_on_range(read_range, read, CommitFlags::empty()) } /// Writes the specified amount of buffer content starting from the target offset in the VMO. @@ -330,31 +375,35 @@ impl Vmo_ { let write_len = reader.remain(); let write_range = offset..(offset + write_len); let mut write_offset = offset % PAGE_SIZE; - - let mut write = move |commit_fn: &mut dyn FnMut() -> Result| { - let frame = commit_fn()?; - frame.writer().skip(write_offset).write_fallible(reader)?; - write_offset = 0; - Ok(()) - }; + let mut write = + move |commit_fn: &mut dyn FnMut() -> core::result::Result| { + let frame = commit_fn()?; + frame + .writer() + .skip(write_offset) + .write_fallible(reader) + .map_err(|e| VmoCommitError::from(e.0))?; + write_offset = 0; + Ok(()) + }; if write_range.len() < PAGE_SIZE { - self.operate_on_range(&write_range, write, CommitFlags::empty())?; + self.operate_on_range(write_range.clone(), write, CommitFlags::empty())?; } else { let temp = write_range.start + PAGE_SIZE - 1; let up_align_start = temp - temp % PAGE_SIZE; let down_align_end = write_range.end - write_range.end % PAGE_SIZE; if write_range.start != up_align_start { let head_range = write_range.start..up_align_start; - self.operate_on_range(&head_range, &mut write, CommitFlags::empty())?; + self.operate_on_range(head_range, &mut write, CommitFlags::empty())?; } if up_align_start != down_align_end { let mid_range = up_align_start..down_align_end; - self.operate_on_range(&mid_range, &mut write, CommitFlags::WILL_OVERWRITE)?; + self.operate_on_range(mid_range, &mut write, CommitFlags::WILL_OVERWRITE)?; } if down_align_end != write_range.end { let tail_range = down_align_end..write_range.end; - self.operate_on_range(&tail_range, &mut write, CommitFlags::empty())?; + self.operate_on_range(tail_range, &mut write, CommitFlags::empty())?; } } @@ -377,7 +426,7 @@ impl Vmo_ { /// Returns the size of current VMO. pub fn size(&self) -> usize { - self.pages.with(|_, size| size) + self.size.load(Ordering::Acquire) } /// Resizes current VMO to target size. @@ -385,40 +434,54 @@ impl Vmo_ { assert!(self.flags.contains(VmoFlags::RESIZABLE)); let new_size = new_size.align_up(PAGE_SIZE); - let Pages::Resizable(ref pages) = self.pages else { - return_errno_with_message!(Errno::EINVAL, "current VMO is not resizable"); - }; + let locked_pages = self.pages.lock(); - let mut lock = pages.lock(); - let old_size = lock.1; + let old_size = self.size(); if new_size == old_size { return Ok(()); } + + self.size.store(new_size, Ordering::Release); + if new_size < old_size { - self.decommit_pages(&mut lock.0, new_size..old_size)?; + self.decommit_pages(locked_pages, new_size..old_size)?; } - lock.1 = new_size; + Ok(()) } - fn decommit_pages(&self, pages: &mut XArray, range: Range) -> Result<()> { + fn decommit_pages( + &self, + mut locked_pages: LockedXArray, + range: Range, + ) -> Result<()> { let page_idx_range = get_page_idx_range(&range); - let mut cursor = pages.cursor_mut(page_idx_range.start as u64); + let mut cursor = locked_pages.cursor_mut(page_idx_range.start as u64); + + let Some(pager) = &self.pager else { + for _ in page_idx_range { + cursor.remove(); + cursor.next(); + } + return Ok(()); + }; + + let mut removed_page_idx = Vec::new(); for page_idx in page_idx_range { - if cursor.remove().is_some() - && let Some(pager) = &self.pager - { - pager.decommit_page(page_idx)?; + if cursor.remove().is_some() { + removed_page_idx.push(page_idx); } cursor.next(); } - Ok(()) - } - /// Determines whether a page is committed. - pub fn is_page_committed(&self, page_idx: usize) -> bool { - self.pages - .with(|pages, _| pages.load(page_idx as u64).is_some()) + drop(cursor); + drop(locked_pages); + + for page_idx in removed_page_idx { + pager.decommit_page(page_idx)?; + } + + Ok(()) } /// Returns the flags of current VMO. @@ -427,13 +490,13 @@ impl Vmo_ { } fn replace(&self, page: UFrame, page_idx: usize) -> Result<()> { - self.pages.with(|pages, size| { - if page_idx >= size / PAGE_SIZE { - return_errno_with_message!(Errno::EINVAL, "the page index is outside of the vmo"); - } - pages.store(page_idx as u64, page); - Ok(()) - }) + let mut locked_pages = self.pages.lock(); + if page_idx >= self.size() / PAGE_SIZE { + return_errno_with_message!(Errno::EINVAL, "the page index is outside of the vmo"); + } + + locked_pages.store(page_idx as u64, page); + Ok(()) } } diff --git a/kernel/src/vm/vmo/options.rs b/kernel/src/vm/vmo/options.rs index c77306292..c3d7121bc 100644 --- a/kernel/src/vm/vmo/options.rs +++ b/kernel/src/vm/vmo/options.rs @@ -4,14 +4,14 @@ //! Options for allocating root and child VMOs. +use core::sync::atomic::AtomicUsize; + use align_ext::AlignExt; use aster_rights::{Rights, TRightSet, TRights}; -use ostd::{ - collections::xarray::XArray, - mm::{FrameAllocOptions, UFrame, USegment}, -}; +use ostd::mm::{FrameAllocOptions, UFrame, USegment}; +use xarray::XArray; -use super::{Pager, Pages, Vmo, VmoFlags}; +use super::{Pager, Vmo, VmoFlags}; use crate::{prelude::*, vm::vmo::Vmo_}; /// Options for allocating a root VMO. @@ -122,18 +122,12 @@ impl VmoOptions> { fn alloc_vmo_(size: usize, flags: VmoFlags, pager: Option>) -> Result { let size = size.align_up(PAGE_SIZE); - let pages = { - let pages = committed_pages_if_continuous(flags, size)?; - if flags.contains(VmoFlags::RESIZABLE) { - Pages::Resizable(Mutex::new((pages, size))) - } else { - Pages::Nonresizable(Mutex::new(pages), size) - } - }; + let pages = committed_pages_if_continuous(flags, size)?; Ok(Vmo_ { pager, flags, pages, + size: AtomicUsize::new(size), }) } @@ -142,13 +136,15 @@ fn committed_pages_if_continuous(flags: VmoFlags, size: usize) -> Result Vmo> { /// Commits a page at specific offset. - pub fn commit_page(&self, offset: usize) -> Result { - self.check_rights(Rights::WRITE)?; - self.0.commit_page(offset) - } - - /// Commits the pages specified in the range (in bytes). /// - /// The range must be within the size of the VMO. - /// - /// The start and end addresses will be rounded down and up to page boundaries. + /// If the commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. /// /// # Access rights /// /// The method requires the Write right. #[require(R > Write)] - pub fn commit(&self, range: Range) -> Result<()> { - self.0.operate_on_range( - &range, - |commit_fn| commit_fn().map(|_| ()), - CommitFlags::empty(), - )?; - Ok(()) + pub fn try_commit_page(&self, offset: usize) -> core::result::Result { + self.check_rights(Rights::WRITE)?; + self.0.try_commit_page(offset) + } + + /// Commits a page at a specific page index. + /// + /// This method may involve I/O operations if the VMO needs to fetch + /// a page from the underlying page cache. + /// + /// # Access rights + /// + /// The method requires the Write right. + #[require(R > Write)] + pub fn commit_on(&self, page_idx: usize, commit_flags: CommitFlags) -> Result { + self.0.commit_on(page_idx, commit_flags) } /// Traverses the indices within a specified range of a VMO sequentially. + /// /// For each index position, you have the option to commit the page as well as /// perform other operations. + /// + /// Once a commit operation needs to perform I/O, it will return a [`VmoCommitError::NeedIo`]. + /// + /// # Access rights + /// + /// The method requires the Write right. #[require(R > Write)] - pub(in crate::vm) fn operate_on_range(&self, range: &Range, operate: F) -> Result<()> + pub(in crate::vm) fn try_operate_on_range( + &self, + range: &Range, + operate: F, + ) -> core::result::Result<(), VmoCommitError> where - F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + F: FnMut( + &mut dyn FnMut() -> core::result::Result, + ) -> core::result::Result<(), VmoCommitError>, { - self.0 - .operate_on_range(range, operate, CommitFlags::empty()) + self.0.try_operate_on_range(range, operate) } /// Decommits the pages specified in the range (in bytes). @@ -95,19 +108,6 @@ impl Vmo> { Vmo(self.0.clone(), self.1) } - /// Creates a new VMO that replicates the original capability, initially representing - /// the same physical pages. - /// Changes to the permissions and commits/replacements of internal pages in the original VMO - /// and the new VMO will not affect each other. - /// - /// # Access rights - /// - /// The method requires the Dup right. - #[require(R > Dup | Write)] - pub fn dup_independent(&self) -> Self { - Vmo(Arc::new(super::Vmo_::clone(&self.0)), self.1) - } - /// Replaces the page at the `page_idx` in the VMO with the input `page`. /// /// # Access rights