diff --git a/kernel/src/process/program_loader/elf/load_elf.rs b/kernel/src/process/program_loader/elf/load_elf.rs index 7d1eb598..3488104e 100644 --- a/kernel/src/process/program_loader/elf/load_elf.rs +++ b/kernel/src/process/program_loader/elf/load_elf.rs @@ -233,7 +233,9 @@ fn base_map_addr(elf: &Elf, root_vmar: &Vmar) -> Result { "executable file does not has loadable sections", ))?; let map_size = elf_size.align_up(PAGE_SIZE); - let vmar_map_options = root_vmar.new_map(map_size, VmPerms::empty())?; + let vmar_map_options = root_vmar + .new_map(map_size, VmPerms::empty())? + .handle_page_faults_around(); vmar_map_options.build() } @@ -333,7 +335,7 @@ fn map_segment_vmo( .vmo_limit(segment_offset + segment_size) .can_overwrite(true); let offset = base_addr + (program_header.virtual_addr as Vaddr).align_down(PAGE_SIZE); - vm_map_options = vm_map_options.offset(offset); + vm_map_options = vm_map_options.offset(offset).handle_page_faults_around(); let map_addr = vm_map_options.build()?; let anonymous_map_size: usize = if total_map_size > segment_size { diff --git a/kernel/src/syscall/mmap.rs b/kernel/src/syscall/mmap.rs index 5e250418..47b9228d 100644 --- a/kernel/src/syscall/mmap.rs +++ b/kernel/src/syscall/mmap.rs @@ -124,7 +124,10 @@ fn do_sys_mmap( .to_dyn() }; - options = options.vmo(vmo).vmo_offset(offset); + options = options + .vmo(vmo) + .vmo_offset(offset) + .handle_page_faults_around(); } options diff --git a/kernel/src/vm/vmar/vm_mapping.rs b/kernel/src/vm/vmar/vm_mapping.rs index be6ba3d4..6fc0a6ff 100644 --- a/kernel/src/vm/vmar/vm_mapping.rs +++ b/kernel/src/vm/vmar/vm_mapping.rs @@ -3,7 +3,10 @@ #![allow(dead_code)] #![allow(unused_variables)] -use core::ops::Range; +use core::{ + cmp::{max, min}, + ops::Range, +}; use align_ext::AlignExt; use aster_rights::Rights; @@ -46,6 +49,8 @@ pub(super) struct VmMapping { /// or are carried through to the underlying file for /// file-backed shared mappings. is_shared: bool, + /// Whether the mapping needs to handle surrounding pages when handling page fault. + handle_page_faults_around: bool, } impl VmMapping { @@ -57,6 +62,7 @@ impl VmMapping { parent: self.parent.clone(), vmo, is_shared: self.is_shared, + handle_page_faults_around: self.handle_page_faults_around, }) } } @@ -96,6 +102,7 @@ impl VmMapping { align, can_overwrite, is_shared, + handle_page_faults_around, } = option; let Vmar(parent_vmar, _) = parent; let map_to_addr = @@ -130,6 +137,7 @@ impl VmMapping { parent: Arc::downgrade(&parent_vmar), vmo, is_shared, + handle_page_faults_around, }) } @@ -218,6 +226,12 @@ impl VmMapping { ) -> Result<()> { let required_perm = if write { VmPerms::WRITE } else { VmPerms::READ }; self.check_perms(&required_perm)?; + + if !write && self.vmo.is_some() && self.handle_page_faults_around { + self.handle_page_faults_around(page_fault_addr)?; + return Ok(()); + } + let page_aligned_addr = page_fault_addr.align_down(PAGE_SIZE); if write && !not_present { @@ -283,6 +297,47 @@ impl VmMapping { } } + fn handle_page_faults_around(&self, page_fault_addr: Vaddr) -> Result<()> { + const SURROUNDING_PAGE_NUM: usize = 16; + const SURROUNDING_PAGE_ADDR_MASK: usize = !(SURROUNDING_PAGE_NUM * PAGE_SIZE - 1); + + let inner = self.inner.lock(); + let vmo_offset = inner.vmo_offset.unwrap(); + let vmo = self.vmo().unwrap(); + let around_page_addr = page_fault_addr & SURROUNDING_PAGE_ADDR_MASK; + let valid_size = min(vmo.size() - vmo_offset, inner.map_size); + + let start_addr = max(around_page_addr, inner.map_to_addr); + let end_addr = min( + start_addr + SURROUNDING_PAGE_NUM * PAGE_SIZE, + inner.map_to_addr + valid_size, + ); + + let vm_perms = inner.perms - VmPerms::WRITE; + let vm_map_options = { PageProperty::new(vm_perms.into(), CachePolicy::Writeback) }; + let parent = self.parent.upgrade().unwrap(); + let vm_space = parent.vm_space(); + let mut cursor = vm_space.cursor_mut(&(start_addr..end_addr))?; + let operate = move |commit_fn: &mut dyn FnMut() -> Result| { + if let VmItem::NotMapped { .. } = cursor.query().unwrap() { + let frame = commit_fn()?; + cursor.map(frame, vm_map_options); + } else { + let next_addr = cursor.virt_addr() + PAGE_SIZE; + if next_addr < end_addr { + let _ = cursor.jump(next_addr); + } + } + Ok(()) + }; + + let start_offset = vmo_offset + start_addr - inner.map_to_addr; + let end_offset = vmo_offset + end_addr - inner.map_to_addr; + vmo.operate_on_range(&(start_offset..end_offset), operate)?; + + Ok(()) + } + /// Protects a specified range of pages in the mapping to the target perms. /// This `VmMapping` will split to maintain its property. /// @@ -313,6 +368,7 @@ impl VmMapping { parent: Arc::downgrade(new_parent), vmo: self.vmo.as_ref().map(|vmo| vmo.dup()).transpose()?, is_shared: self.is_shared, + handle_page_faults_around: self.handle_page_faults_around, }) } @@ -577,6 +633,8 @@ pub struct VmarMapOptions { can_overwrite: bool, // Whether the mapping is mapped with `MAP_SHARED` is_shared: bool, + // Whether the mapping needs to handle surrounding pages when handling page fault. + handle_page_faults_around: bool, } impl VmarMapOptions { @@ -598,6 +656,7 @@ impl VmarMapOptions { align: PAGE_SIZE, can_overwrite: false, is_shared: false, + handle_page_faults_around: false, } } @@ -684,6 +743,12 @@ impl VmarMapOptions { self } + /// Sets the mapping to handle surrounding pages when handling page fault. + pub fn handle_page_faults_around(mut self) -> Self { + self.handle_page_faults_around = true; + self + } + /// Creates the mapping. /// /// All options will be checked at this point. @@ -786,6 +851,18 @@ impl MappedVmo { self.vmo.commit_page(page_idx * PAGE_SIZE) } + /// Traverses the indices within a specified range of a VMO sequentially. + /// For each index position, you have the option to commit the page as well as + /// perform other operations. + fn operate_on_range(&self, range: &Range, operate: F) -> Result<()> + where + F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + { + debug_assert!(self.range.start <= range.start && self.range.end >= range.end); + + self.vmo.operate_on_range(range, operate) + } + /// Duplicates the capability. pub fn dup(&self) -> Result { Ok(Self { @@ -793,4 +870,9 @@ impl MappedVmo { range: self.range.clone(), }) } + + /// Returns the size (in bytes) of a VMO. + pub fn size(&self) -> usize { + self.vmo.size() + } } diff --git a/kernel/src/vm/vmo/dyn_cap.rs b/kernel/src/vm/vmo/dyn_cap.rs index 1b1ccdc4..245657ff 100644 --- a/kernel/src/vm/vmo/dyn_cap.rs +++ b/kernel/src/vm/vmo/dyn_cap.rs @@ -26,11 +26,26 @@ impl Vmo { /// The method requires the Write right. pub fn commit(&self, range: Range) -> Result<()> { self.check_rights(Rights::WRITE)?; - self.0 - .commit_and_operate(&range, |_| Ok(()), CommitFlags::empty())?; + self.0.operate_on_range( + &range, + |commit_fn| commit_fn().map(|_| ()), + CommitFlags::empty(), + )?; Ok(()) } + /// Traverses the indices within a specified range of a VMO sequentially. + /// For each index position, you have the option to commit the page as well as + /// perform other operations. + pub(in crate::vm) fn operate_on_range(&self, range: &Range, operate: F) -> Result<()> + where + F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + { + self.check_rights(Rights::WRITE)?; + self.0 + .operate_on_range(range, operate, CommitFlags::empty()) + } + /// Decommits the pages specified in the range (in bytes). /// /// The range must be within the size of the VMO. diff --git a/kernel/src/vm/vmo/mod.rs b/kernel/src/vm/vmo/mod.rs index 2b095d17..b175ac39 100644 --- a/kernel/src/vm/vmo/mod.rs +++ b/kernel/src/vm/vmo/mod.rs @@ -260,16 +260,17 @@ impl Vmo_ { }) } - /// Commits a range of pages in the VMO, and perform the operation - /// on each page in the range in turn. - pub fn commit_and_operate( + /// Traverses the indices within a specified range of a VMO sequentially. + /// For each index position, you have the option to commit the page as well as + /// perform other operations. + pub fn operate_on_range( &self, range: &Range, mut operate: F, commit_flags: CommitFlags, ) -> Result<()> where - F: FnMut(Frame) -> Result<()>, + F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, { self.pages.with(|pages, size| { if range.end > size { @@ -279,8 +280,8 @@ impl Vmo_ { let page_idx_range = get_page_idx_range(range); let mut cursor = pages.cursor_mut(page_idx_range.start as u64); for page_idx in page_idx_range { - let committed_page = self.commit_with_cursor(&mut cursor, commit_flags)?; - operate(committed_page)?; + let mut commit_fn = || self.commit_with_cursor(&mut cursor, commit_flags); + operate(&mut commit_fn)?; cursor.next(); } Ok(()) @@ -305,13 +306,14 @@ impl Vmo_ { let read_range = offset..(offset + read_len); let mut read_offset = offset % PAGE_SIZE; - let read = move |page: Frame| -> Result<()> { - page.reader().skip(read_offset).read_fallible(writer)?; + let read = move |commit_fn: &mut dyn FnMut() -> Result| { + let frame = commit_fn()?; + frame.reader().skip(read_offset).read_fallible(writer)?; read_offset = 0; Ok(()) }; - self.commit_and_operate(&read_range, read, CommitFlags::empty()) + self.operate_on_range(&read_range, read, CommitFlags::empty()) } /// Writes the specified amount of buffer content starting from the target offset in the VMO. @@ -320,29 +322,30 @@ impl Vmo_ { let write_range = offset..(offset + write_len); let mut write_offset = offset % PAGE_SIZE; - let mut write = move |page: Frame| -> Result<()> { - page.writer().skip(write_offset).write_fallible(reader)?; + let mut write = move |commit_fn: &mut dyn FnMut() -> Result| { + let frame = commit_fn()?; + frame.writer().skip(write_offset).write_fallible(reader)?; write_offset = 0; Ok(()) }; if write_range.len() < PAGE_SIZE { - self.commit_and_operate(&write_range, write, CommitFlags::empty())?; + self.operate_on_range(&write_range, write, CommitFlags::empty())?; } else { let temp = write_range.start + PAGE_SIZE - 1; let up_align_start = temp - temp % PAGE_SIZE; let down_align_end = write_range.end - write_range.end % PAGE_SIZE; if write_range.start != up_align_start { let head_range = write_range.start..up_align_start; - self.commit_and_operate(&head_range, &mut write, CommitFlags::empty())?; + self.operate_on_range(&head_range, &mut write, CommitFlags::empty())?; } if up_align_start != down_align_end { let mid_range = up_align_start..down_align_end; - self.commit_and_operate(&mid_range, &mut write, CommitFlags::WILL_OVERWRITE)?; + self.operate_on_range(&mid_range, &mut write, CommitFlags::WILL_OVERWRITE)?; } if down_align_end != write_range.end { let tail_range = down_align_end..write_range.end; - self.commit_and_operate(&tail_range, &mut write, CommitFlags::empty())?; + self.operate_on_range(&tail_range, &mut write, CommitFlags::empty())?; } } diff --git a/kernel/src/vm/vmo/static_cap.rs b/kernel/src/vm/vmo/static_cap.rs index e623a068..5f484239 100644 --- a/kernel/src/vm/vmo/static_cap.rs +++ b/kernel/src/vm/vmo/static_cap.rs @@ -27,11 +27,26 @@ impl Vmo> { /// The method requires the Write right. #[require(R > Write)] pub fn commit(&self, range: Range) -> Result<()> { - self.0 - .commit_and_operate(&range, |_| Ok(()), CommitFlags::empty())?; + self.0.operate_on_range( + &range, + |commit_fn| commit_fn().map(|_| ()), + CommitFlags::empty(), + )?; Ok(()) } + /// Traverses the indices within a specified range of a VMO sequentially. + /// For each index position, you have the option to commit the page as well as + /// perform other operations. + #[require(R > Write)] + pub(in crate::vm) fn operate_on_range(&self, range: &Range, operate: F) -> Result<()> + where + F: FnMut(&mut dyn FnMut() -> Result) -> Result<()>, + { + self.0 + .operate_on_range(range, operate, CommitFlags::empty()) + } + /// Decommits the pages specified in the range (in bytes). /// /// The range must be within the size of the VMO. diff --git a/ostd/src/mm/vm_space.rs b/ostd/src/mm/vm_space.rs index fcfdf2a8..52857fdb 100644 --- a/ostd/src/mm/vm_space.rs +++ b/ostd/src/mm/vm_space.rs @@ -341,7 +341,7 @@ impl CursorMut<'_> { /// Jump to the virtual address. /// /// This is the same as [`Cursor::jump`]. - pub fn jump(&mut self, va: Vaddr) -> Result<()>{ + pub fn jump(&mut self, va: Vaddr) -> Result<()> { self.0.jump(va)?; Ok(()) }