From 38ee2cafcddcd90447de53aa68f0d26cac93867b Mon Sep 17 00:00:00 2001 From: Jianfeng Jiang Date: Mon, 17 Jul 2023 14:36:27 +0800 Subject: [PATCH] Inherit vmo at page granularity --- Cargo.lock | 1 - framework/jinux-frame/src/vm/frame.rs | 4 + services/libs/jinux-std/Cargo.toml | 5 - .../process/program_loader/elf/load_elf.rs | 2 +- .../libs/jinux-std/src/vm/vmar/vm_mapping.rs | 20 +- services/libs/jinux-std/src/vm/vmo/mod.rs | 233 ++++++------------ services/libs/jinux-std/src/vm/vmo/options.rs | 64 ++--- services/libs/jinux-util/src/slot_vec.rs | 7 + 8 files changed, 128 insertions(+), 208 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8e6c2135..172c61fc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -573,7 +573,6 @@ dependencies = [ "align_ext", "ascii", "bitflags", - "bitvec", "controlled", "core2", "cpio-decoder", diff --git a/framework/jinux-frame/src/vm/frame.rs b/framework/jinux-frame/src/vm/frame.rs index dc116cb6d..89931667f 100644 --- a/framework/jinux-frame/src/vm/frame.rs +++ b/framework/jinux-frame/src/vm/frame.rs @@ -78,6 +78,10 @@ impl VmFrameVec { Self(Vec::new()) } + pub fn new_with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + /// Pushs a new frame to the collection. pub fn push(&mut self, new_frame: VmFrame) { self.0.push(new_frame); diff --git a/services/libs/jinux-std/Cargo.toml b/services/libs/jinux-std/Cargo.toml index efbf6568a..684cc5453 100644 --- a/services/libs/jinux-std/Cargo.toml +++ b/services/libs/jinux-std/Cargo.toml @@ -47,8 +47,3 @@ getrandom = { version = "0.2.10", default-features = false, features = ["rdrand" [dependencies.lazy_static] version = "1.0" features = ["spin_no_std"] - -[dependencies.bitvec] -version = "1" -default-features = false -features = ["atomic", "alloc"] diff --git a/services/libs/jinux-std/src/process/program_loader/elf/load_elf.rs b/services/libs/jinux-std/src/process/program_loader/elf/load_elf.rs index f9f4ede71..8c6199e7e 100644 --- a/services/libs/jinux-std/src/process/program_loader/elf/load_elf.rs +++ b/services/libs/jinux-std/src/process/program_loader/elf/load_elf.rs @@ -19,7 +19,7 @@ use super::elf_file::Elf; /// load elf to the root vmar. this function will /// 1. read the vaddr of each segment to get all elf pages. -/// 2. create a vmo for each elf segment, create a backup pager for each segment. Then map the vmo to the root vmar. +/// 2. create a vmo for each elf segment, create a pager for each segment. Then map the vmo to the root vmar. /// 3. write proper content to the init stack. pub fn load_elf_to_root_vmar( root_vmar: &Vmar, diff --git a/services/libs/jinux-std/src/vm/vmar/vm_mapping.rs b/services/libs/jinux-std/src/vm/vmar/vm_mapping.rs index d71159fa7..fbb700f6e 100644 --- a/services/libs/jinux-std/src/vm/vmar/vm_mapping.rs +++ b/services/libs/jinux-std/src/vm/vmar/vm_mapping.rs @@ -1,6 +1,6 @@ use crate::prelude::*; use core::ops::Range; -use jinux_frame::vm::VmMapOptions; +use jinux_frame::vm::{VmFrame, VmMapOptions}; use jinux_frame::vm::{VmFrameVec, VmIo, VmPerm}; use spin::Mutex; @@ -42,7 +42,7 @@ impl VmMapping { struct VmMappingInner { /// The map offset of the vmo, in bytes. vmo_offset: usize, - /// The size of mapping, in bytes. The map size can even be larger than the size of backup vmo. + /// The size of mapping, in bytes. The map size can even be larger than the size of vmo. /// Those pages outside vmo range cannot be read or write. map_size: usize, /// The base address relative to the root vmar where the vmo is mapped. @@ -120,7 +120,7 @@ impl VmMapping { pub(super) fn map_one_page( &self, page_idx: usize, - frames: VmFrameVec, + frame: VmFrame, forbid_write_access: bool, ) -> Result<()> { let parent = self.parent.upgrade().unwrap(); @@ -138,7 +138,7 @@ impl VmMapping { if self.vmo.is_cow_child() && vm_space.is_mapped(map_addr) { vm_space.unmap(&(map_addr..(map_addr + PAGE_SIZE))).unwrap(); } - vm_space.map(frames, &vm_map_options)?; + vm_space.map(VmFrameVec::from_one_frame(frame), &vm_map_options)?; self.inner.lock().mapped_pages.insert(page_idx); Ok(()) } @@ -226,15 +226,15 @@ impl VmMapping { } self.check_perm(&page_idx, write)?; - // get the backup frame for page - let frames = self.vmo.get_backup_frame(page_idx, write, true)?; + // get the committed frame for page + let frame = self.vmo.get_committed_frame(page_idx, write)?; // map the page if self.vmo.is_cow_child() && !write { // Since the read access triggers page fault, the child vmo does not commit a new frame ever. // Note the frame is from parent, so the map must forbid write access. - self.map_one_page(page_idx, frames, true) + self.map_one_page(page_idx, frame, true) } else { - self.map_one_page(page_idx, frames, false) + self.map_one_page(page_idx, frame, false) } } @@ -301,8 +301,8 @@ impl VmMapping { let map_addr = (page_idx - start_page_idx) * PAGE_SIZE + map_to_addr; vm_map_options.addr(Some(map_addr)); vm_map_options.perm(vm_perm); - if let Ok(frames) = child_vmo.get_backup_frame(page_idx, false, false) { - vm_space.map(frames, &vm_map_options)?; + if let Ok(frame) = child_vmo.get_committed_frame(page_idx, false) { + vm_space.map(VmFrameVec::from_one_frame(frame), &vm_map_options)?; mapped_pages.insert(page_idx); } } diff --git a/services/libs/jinux-std/src/vm/vmo/mod.rs b/services/libs/jinux-std/src/vm/vmo/mod.rs index 2098ba77c..5f8a739c1 100644 --- a/services/libs/jinux-std/src/vm/vmo/mod.rs +++ b/services/libs/jinux-std/src/vm/vmo/mod.rs @@ -3,9 +3,7 @@ use core::ops::Range; use align_ext::AlignExt; -use bitvec::bitvec; -use bitvec::vec::BitVec; -use jinux_frame::vm::{VmAllocOptions, VmFrameVec, VmIo}; +use jinux_frame::vm::{VmAllocOptions, VmFrame, VmFrameVec, VmIo}; use jinux_rights::Rights; use crate::prelude::*; @@ -135,114 +133,16 @@ pub(super) struct Vmo_ { } struct VmoInner { - /// The backup pager pager: Option>, /// size, in bytes size: usize, - /// The pages committed. The key is the page index, the value is the backup frame. - committed_pages: BTreeMap, + /// The pages committed. The key is the page index, the value is the committed frame. + committed_pages: BTreeMap, /// The pages from the parent that current vmo can access. The pages can only be inherited when create childs vmo. /// We store the page index range - inherited_pages: Option, -} - -/// Pages inherited from parent -struct InheritedPages { - /// Parent vmo. - parent: Option>, - /// The page index range in child vmo. The pages inside these range are initially inherited from parent vmo. - /// The range includes the start page, but not including the end page - page_range: Range, - /// The page index offset in parent vmo. That is to say, the page with index `idx` in child vmo corrsponds to - /// page with index `idx + parent_page_idx_offset` in parent vmo - parent_page_idx_offset: usize, - is_copy_on_write: bool, - /// The pages already committed by child - committed_pages: BitVec, -} - -impl InheritedPages { - pub fn new( - parent: Arc, - page_range: Range, - parent_page_idx_offset: usize, - is_copy_on_write: bool, - ) -> Self { - let committed_pages = bitvec![0; page_range.len()]; - Self { - parent: Some(parent), - page_range, - parent_page_idx_offset, - is_copy_on_write, - committed_pages, - } - } - - fn contains_page(&self, page_idx: usize) -> bool { - self.page_range.start <= page_idx && page_idx < self.page_range.end - } - - fn parent_page_idx(&self, child_page_idx: usize) -> Option { - if self.contains_page(child_page_idx) { - Some(child_page_idx + self.parent_page_idx_offset) - } else { - None - } - } - - fn get_frame_from_parent( - &self, - page_idx: usize, - write_page: bool, - commit_if_none: bool, - ) -> Result { - let Some(parent_page_idx) = self.parent_page_idx(page_idx) else { - if self.is_copy_on_write { - let options = VmAllocOptions::new(1); - return Ok(VmFrameVec::allocate(&options)?); - } - return_errno_with_message!(Errno::EINVAL, "the page is not inherited from parent"); - }; - match (self.is_copy_on_write, write_page) { - (false, _) | (true, false) => { - debug_assert!(self.parent.is_some()); - let parent = self.parent.as_ref().unwrap(); - parent.get_backup_frame(parent_page_idx, write_page, commit_if_none) - } - (true, true) => { - debug_assert!(self.parent.is_some()); - let parent = self.parent.as_ref().unwrap(); - let tmp_buffer = { - let mut buffer = Box::new([0u8; PAGE_SIZE]); - parent.read_bytes(parent_page_idx * PAGE_SIZE, &mut *buffer)?; - buffer - }; - let frames = { - let options = VmAllocOptions::new(1); - VmFrameVec::allocate(&options)? - }; - frames.write_bytes(0, &*tmp_buffer)?; - Ok(frames) - } - } - } - - fn should_child_commit_page(&self, write_page: bool) -> bool { - !self.is_copy_on_write || (self.is_copy_on_write && write_page) - } - - fn mark_page_as_commited(&mut self, page_idx: usize) { - if !self.contains_page(page_idx) { - return; - } - let idx = page_idx - self.page_range.start; - debug_assert_eq!(self.committed_pages[idx], false); - debug_assert!(self.parent.is_some()); - self.committed_pages.set(idx, true); - if self.committed_pages.count_ones() == self.page_range.len() { - self.parent.take(); - } - } + inherited_pages: Option>, + /// Whether the vmo is copy on write child. + is_cow: bool, } impl VmoInner { @@ -252,17 +152,14 @@ impl VmoInner { if self.committed_pages.contains_key(&page_idx) { return Ok(()); } - let frames = match &self.pager { + let frame = match &self.pager { None => { let vm_alloc_option = VmAllocOptions::new(1); - VmFrameVec::allocate(&vm_alloc_option)? - } - Some(pager) => { - let frame = pager.commit_page(offset)?; - VmFrameVec::from_one_frame(frame) + VmFrameVec::allocate(&vm_alloc_option)?.pop().unwrap() } + Some(pager) => pager.commit_page(offset)?, }; - self.committed_pages.insert(page_idx, frames); + self.insert_frame(page_idx, frame); Ok(()) } @@ -276,17 +173,12 @@ impl VmoInner { Ok(()) } - fn insert_frame(&mut self, page_idx: usize, frame: VmFrameVec) { + fn insert_frame(&mut self, page_idx: usize, frame: VmFrame) { debug_assert!(!self.committed_pages.contains_key(&page_idx)); self.committed_pages.insert(page_idx, frame); } - fn get_backup_frame( - &mut self, - page_idx: usize, - write_page: bool, - commit_if_none: bool, - ) -> Result { + fn get_committed_frame(&mut self, page_idx: usize, write_page: bool) -> Result { // if the page is already commit, return the committed page. if let Some(frames) = self.committed_pages.get(&page_idx) { return Ok(frames.clone()); @@ -295,27 +187,50 @@ impl VmoInner { // The vmo is not child if self.inherited_pages.is_none() { self.commit_page(page_idx * PAGE_SIZE)?; - let frames = self.committed_pages.get(&page_idx).unwrap().clone(); - return Ok(frames); + let frame = self.committed_pages.get(&page_idx).unwrap().clone(); + return Ok(frame); } - let inherited_pages = self.inherited_pages.as_mut().unwrap(); - let frame = inherited_pages.get_frame_from_parent(page_idx, write_page, commit_if_none)?; + let frame = self.get_inherited_frame_or_alloc(page_idx, write_page)?; - if inherited_pages.should_child_commit_page(write_page) { - inherited_pages.mark_page_as_commited(page_idx); + if !self.should_share_frame_with_parent(write_page) { self.insert_frame(page_idx, frame.clone()); } Ok(frame) } - fn is_cow_child(&self) -> bool { - if let Some(inherited_pages) = &self.inherited_pages { - inherited_pages.is_copy_on_write - } else { - false + fn get_inherited_frame_or_alloc(&self, page_idx: usize, write_page: bool) -> Result { + let inherited_frames = self.inherited_pages.as_ref().unwrap(); + + if page_idx >= inherited_frames.len() { + if self.is_cow { + let options = VmAllocOptions::new(1); + return Ok(VmFrameVec::allocate(&options)?.pop().unwrap()); + } + return_errno_with_message!(Errno::EINVAL, "the page is not inherited from parent"); } + + let inherited_frame = inherited_frames.get(page_idx).unwrap().clone(); + + if self.should_share_frame_with_parent(write_page) { + return Ok(inherited_frame); + } + + let frame = { + let options = VmAllocOptions::new(1); + VmFrameVec::allocate(&options)?.pop().unwrap() + }; + frame.write_bytes(0, &*tmp_buffer)?; + Ok(frame) + } + + fn is_cow_child(&self) -> bool { + self.is_cow + } + + fn should_share_frame_with_parent(&self, write_page: bool) -> bool { + !self.is_cow || (self.is_cow && !write_page) } } @@ -366,25 +281,18 @@ impl Vmo_ { /// Ensure all pages inside range are backed up vm frames, returns the frames. fn ensure_all_pages_exist(&self, range: &Range, write_page: bool) -> Result { let page_idx_range = get_page_idx_range(range); - let mut frames = VmFrameVec::empty(); + let mut frames = VmFrameVec::new_with_capacity(page_idx_range.len()); for page_idx in page_idx_range { - let mut page_frame = self.get_backup_frame(page_idx, write_page, true)?; - frames.append(&mut page_frame)?; + let page_frame = self.get_committed_frame(page_idx, write_page)?; + frames.push(page_frame); } Ok(frames) } - /// Get the backup frame for a page. If commit_if_none is set, we will commit a new page for the page - /// if the page does not have a backup frame. - fn get_backup_frame( - &self, - page_idx: usize, - write_page: bool, - commit_if_none: bool, - ) -> Result { - self.inner - .lock() - .get_backup_frame(page_idx, write_page, commit_if_none) + /// Get the frame for a page. If commit_if_none is set, we will commit a new page for the page + /// if the page is not committed. + fn get_committed_frame(&self, page_idx: usize, write_page: bool) -> Result { + self.inner.lock().get_committed_frame(page_idx, write_page) } pub fn write_bytes(&self, offset: usize, buf: &[u8]) -> Result<()> { @@ -451,22 +359,12 @@ impl Vmo { } /// return whether a page is already committed - pub fn has_backup_frame(&self, page_idx: usize) -> bool { - if let Ok(_) = self.0.get_backup_frame(page_idx, false, false) { - true - } else { - false - } + pub fn is_page_committed(&self, page_idx: usize) -> bool { + self.0.page_commited(page_idx) } - pub fn get_backup_frame( - &self, - page_idx: usize, - write_page: bool, - commit_if_none: bool, - ) -> Result { - self.0 - .get_backup_frame(page_idx, write_page, commit_if_none) + pub fn get_committed_frame(&self, page_idx: usize, write_page: bool) -> Result { + self.0.get_committed_frame(page_idx, write_page) } pub fn is_cow_child(&self) -> bool { @@ -480,3 +378,20 @@ pub fn get_page_idx_range(vmo_offset_range: &Range) -> Range { let end = vmo_offset_range.end.align_up(PAGE_SIZE); (start / PAGE_SIZE)..(end / PAGE_SIZE) } + +pub(super) fn get_inherited_frames_from_parent( + parent: Arc, + num_pages: usize, + parent_page_idx_offset: usize, + is_cow: bool, +) -> Vec { + let mut inherited_frames = Vec::with_capacity(num_pages); + for page_idx in 0..num_pages { + let parent_page_idx = page_idx + parent_page_idx_offset; + let inherited_frame = parent + .get_committed_frame(parent_page_idx, !is_cow) + .unwrap(); + inherited_frames.push(inherited_frame); + } + inherited_frames +} diff --git a/services/libs/jinux-std/src/vm/vmo/options.rs b/services/libs/jinux-std/src/vm/vmo/options.rs index fa554cb05..ef5adc87f 100644 --- a/services/libs/jinux-std/src/vm/vmo/options.rs +++ b/services/libs/jinux-std/src/vm/vmo/options.rs @@ -4,13 +4,13 @@ use core::marker::PhantomData; use core::ops::Range; use align_ext::AlignExt; -use jinux_frame::vm::{VmAllocOptions, VmFrameVec}; +use jinux_frame::vm::{VmAllocOptions, VmFrame, VmFrameVec}; use jinux_rights_proc::require; use typeflags_util::{SetExtend, SetExtendOp}; use crate::prelude::*; -use crate::vm::vmo::InheritedPages; +use crate::vm::vmo::get_inherited_frames_from_parent; use crate::vm::vmo::{VmoInner, Vmo_}; use jinux_rights::{Dup, Rights, TRightSet, TRights, Write}; @@ -131,6 +131,7 @@ fn alloc_vmo_(size: usize, flags: VmoFlags, pager: Option>) -> Re size, committed_pages, inherited_pages: None, + is_cow: false, }; Ok(Vmo_ { flags, @@ -138,10 +139,7 @@ fn alloc_vmo_(size: usize, flags: VmoFlags, pager: Option>) -> Re }) } -fn committed_pages_if_continuous( - flags: VmoFlags, - size: usize, -) -> Result> { +fn committed_pages_if_continuous(flags: VmoFlags, size: usize) -> Result> { if flags.contains(VmoFlags::CONTIGUOUS) { // if the vmo is continuous, we need to allocate frames for the vmo let frames_num = size / PAGE_SIZE; @@ -150,7 +148,7 @@ fn committed_pages_if_continuous( let frames = VmFrameVec::allocate(&vm_alloc_option)?; let mut committed_pages = BTreeMap::new(); for (idx, frame) in frames.into_iter().enumerate() { - committed_pages.insert(idx * PAGE_SIZE, VmFrameVec::from_one_frame(frame)); + committed_pages.insert(idx * PAGE_SIZE, frame); } Ok(committed_pages) } else { @@ -454,27 +452,32 @@ fn alloc_child_vmo_( return_errno_with_message!(Errno::EINVAL, "vmo range does not aligned with PAGE_SIZE"); } let parent_vmo_size = parent_vmo_.size(); - let parent_vmo_inner = parent_vmo_.inner.lock(); - let is_copy_on_write = match child_type { - ChildType::Slice => { - // A slice child should be inside parent vmo's range - debug_assert!(child_vmo_end <= parent_vmo_inner.size); - if child_vmo_end > parent_vmo_inner.size { - return_errno_with_message!( - Errno::EINVAL, - "slice child vmo cannot exceed parent vmo's size" - ); + let is_cow = { + let parent_vmo_inner = parent_vmo_.inner.lock(); + match child_type { + ChildType::Slice => { + // A slice child should be inside parent vmo's range + debug_assert!(child_vmo_end <= parent_vmo_inner.size); + if child_vmo_end > parent_vmo_inner.size { + return_errno_with_message!( + Errno::EINVAL, + "slice child vmo cannot exceed parent vmo's size" + ); + } + false } - false - } - ChildType::Cow => { - // A copy on Write child should intersect with parent vmo - debug_assert!(range.start <= parent_vmo_inner.size); - if range.start > parent_vmo_inner.size { - return_errno_with_message!(Errno::EINVAL, "COW vmo should overlap with its parent"); + ChildType::Cow => { + // A copy on Write child should intersect with parent vmo + debug_assert!(range.start <= parent_vmo_inner.size); + if range.start > parent_vmo_inner.size { + return_errno_with_message!( + Errno::EINVAL, + "COW vmo should overlap with its parent" + ); + } + true } - true } }; let parent_page_idx_offset = range.start / PAGE_SIZE; @@ -484,18 +487,15 @@ fn alloc_child_vmo_( } else { 0 }; - let inherited_end_page_idx = cow_size / PAGE_SIZE; - let inherited_pages = InheritedPages::new( - parent_vmo_.clone(), - 0..inherited_end_page_idx, - parent_page_idx_offset, - is_copy_on_write, - ); + let num_pages = cow_size / PAGE_SIZE; + let inherited_pages = + get_inherited_frames_from_parent(parent_vmo_, num_pages, parent_page_idx_offset, is_cow); let vmo_inner = VmoInner { pager: None, size: child_vmo_end - child_vmo_start, committed_pages: BTreeMap::new(), inherited_pages: Some(inherited_pages), + is_cow, }; Ok(Vmo_ { flags: child_flags, diff --git a/services/libs/jinux-util/src/slot_vec.rs b/services/libs/jinux-util/src/slot_vec.rs index 5c3e9faef..027f3faec 100644 --- a/services/libs/jinux-util/src/slot_vec.rs +++ b/services/libs/jinux-util/src/slot_vec.rs @@ -20,6 +20,13 @@ impl SlotVec { } } + pub fn with_capacity(capacity: usize) -> Self { + Self { + slots: Vec::with_capacity(capacity), + num_occupied: 0, + } + } + /// Return `true` if the vector contains no items. pub fn is_empty(&self) -> bool { self.num_occupied == 0