diff --git a/Cargo.lock b/Cargo.lock index 2f113d04a..05d45c08e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -83,6 +83,7 @@ dependencies = [ "align_ext", "aster-util", "bitflags 1.3.2", + "bitvec", "component", "int-to-c-enum", "log", diff --git a/kernel/comps/block/Cargo.toml b/kernel/comps/block/Cargo.toml index 86fc0056d..6dc5351eb 100644 --- a/kernel/comps/block/Cargo.toml +++ b/kernel/comps/block/Cargo.toml @@ -15,5 +15,6 @@ int-to-c-enum = { path = "../../libs/int-to-c-enum" } component = { path = "../../libs/comp-sys/component" } log = "0.4" static_assertions = "1.1.0" +bitvec = { version = "1.0.1", default-features = false, features = ["alloc"] } [features] diff --git a/kernel/comps/block/src/bio.rs b/kernel/comps/block/src/bio.rs index 996c6a5fa..7540d3b42 100644 --- a/kernel/comps/block/src/bio.rs +++ b/kernel/comps/block/src/bio.rs @@ -1,15 +1,20 @@ // SPDX-License-Identifier: MPL-2.0 use align_ext::AlignExt; -use aster_util::segment_slice::SegmentSlice; +use bitvec::array::BitArray; use int_to_c_enum::TryFromInt; use ostd::{ - mm::{Frame, Infallible, VmReader, VmWriter}, - sync::WaitQueue, + mm::{ + DmaDirection, DmaStream, DmaStreamSlice, FrameAllocOptions, Infallible, Segment, VmIo, + VmReader, VmWriter, + }, + sync::{SpinLock, WaitQueue}, + Error, }; +use spin::Once; use super::{id::Sid, BlockDevice}; -use crate::prelude::*; +use crate::{prelude::*, BLOCK_SIZE, SECTOR_SIZE}; /// The unit for block I/O. /// @@ -361,88 +366,349 @@ pub enum BioStatus { IoError = 5, } -/// `BioSegment` is a smallest memory unit in block I/O. -/// -/// It is a contiguous memory region that contains multiple sectors. +/// `BioSegment` is the basic memory unit of a block I/O request. #[derive(Debug, Clone)] pub struct BioSegment { - /// The contiguous pages on which this segment resides. - pages: SegmentSlice, - /// The starting offset (in bytes) within the first page. - /// The offset should always be aligned to the sector size and - /// must not exceed the size of a single page. - offset: AlignedUsize, - /// The total length (in bytes). - /// The length can span multiple pages and should be aligned to - /// the sector size. - len: AlignedUsize, + inner: Arc, } -const SECTOR_SIZE: u16 = super::SECTOR_SIZE as u16; +/// The inner part of `BioSegment`. +// TODO: Decouple `BioSegmentInner` with DMA-related buffers. +#[derive(Debug, Clone)] +struct BioSegmentInner { + /// Internal DMA slice. + dma_slice: DmaStreamSlice, + /// Whether the segment is allocated from the pool. + from_pool: bool, +} + +/// The direction of a bio request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BioDirection { + /// Read from the backed block device. + FromDevice, + /// Write to the backed block device. + ToDevice, +} impl<'a> BioSegment { - /// Constructs a new `BioSegment` from `Segment`. - pub fn from_segment(segment: SegmentSlice, offset: usize, len: usize) -> Self { - assert!(offset + len <= segment.nbytes()); + /// Allocates a new `BioSegment` with the wanted blocks count and + /// the bio direction. + pub fn alloc(nblocks: usize, direction: BioDirection) -> Self { + Self::alloc_inner(nblocks, 0, nblocks * BLOCK_SIZE, direction) + } + + /// The inner function that do the real segment allocation. + /// + /// Support two extended parameters: + /// 1. `offset_within_first_block`: the offset (in bytes) within the first block. + /// 2. `len`: the exact length (in bytes) of the wanted segment. (May + /// less than `nblocks * BLOCK_SIZE`) + /// + /// # Panics + /// + /// If the `offset_within_first_block` or `len` is not sector aligned, + /// this method will panic. + pub(super) fn alloc_inner( + nblocks: usize, + offset_within_first_block: usize, + len: usize, + direction: BioDirection, + ) -> Self { + assert!( + is_sector_aligned(offset_within_first_block) + && offset_within_first_block < BLOCK_SIZE + && is_sector_aligned(len) + && offset_within_first_block + len <= nblocks * BLOCK_SIZE + ); + + // The target segment is whether from the pool or newly-allocated + let bio_segment_inner = target_pool(direction) + .and_then(|pool| pool.alloc(nblocks, offset_within_first_block, len)) + .unwrap_or_else(|| { + let segment = FrameAllocOptions::new(nblocks) + .uninit(true) + .alloc_contiguous() + .unwrap(); + let dma_stream = DmaStream::map(segment, direction.into(), false).unwrap(); + BioSegmentInner { + dma_slice: DmaStreamSlice::new(dma_stream, offset_within_first_block, len), + from_pool: false, + } + }); + Self { - pages: segment.range(frame_range(&(offset..offset + len))), - offset: AlignedUsize::::new(offset % super::BLOCK_SIZE).unwrap(), - len: AlignedUsize::::new(len).unwrap(), + inner: Arc::new(bio_segment_inner), } } - /// Constructs a new `BioSegment` from `Frame`. - pub fn from_frame(frame: Frame, offset: usize, len: usize) -> Self { - assert!(offset + len <= super::BLOCK_SIZE); - + /// Constructs a new `BioSegment` with a given `Segment` and the bio direction. + pub fn new_from_segment(segment: Segment, direction: BioDirection) -> Self { + let len = segment.nbytes(); + let dma_stream = DmaStream::map(segment, direction.into(), false).unwrap(); Self { - pages: SegmentSlice::from(frame), - offset: AlignedUsize::::new(offset).unwrap(), - len: AlignedUsize::::new(len).unwrap(), + inner: Arc::new(BioSegmentInner { + dma_slice: DmaStreamSlice::new(dma_stream, 0, len), + from_pool: false, + }), } } - /// Returns the number of sectors. - pub fn nsectors(&self) -> Sid { - Sid::from_offset(self.len.value()) - } - /// Returns the number of bytes. pub fn nbytes(&self) -> usize { - self.len.value() + self.inner.dma_slice.nbytes() } - /// Returns the offset (in bytes) within the first page. - pub fn offset(&self) -> usize { - self.offset.value() + /// Returns the number of sectors. + pub fn nsectors(&self) -> Sid { + Sid::from_offset(self.nbytes()) } - /// Returns the contiguous pages on which this segment resides. - pub fn pages(&self) -> &SegmentSlice { - &self.pages + /// Returns the number of blocks. + pub fn nblocks(&self) -> usize { + self.nbytes().align_up(BLOCK_SIZE) / BLOCK_SIZE + } + + /// Returns the offset (in bytes) within the first block. + pub fn offset_within_first_block(&self) -> usize { + self.inner.dma_slice.offset() % BLOCK_SIZE + } + + /// Returns the inner DMA slice. + pub fn inner_dma_slice(&self) -> &DmaStreamSlice { + &self.inner.dma_slice + } + + /// Returns the inner VM segment. + #[cfg(ktest)] + pub fn inner_segment(&self) -> &Segment { + self.inner.dma_slice.stream().vm_segment() } /// Returns a reader to read data from it. - pub fn reader(&'a self) -> VmReader<'a, Infallible> { - self.pages - .reader() - .skip(self.offset.value()) - .limit(self.len.value()) + pub fn reader(&'a self) -> Result, Error> { + self.inner.dma_slice.reader() } /// Returns a writer to write data into it. - pub fn writer(&'a self) -> VmWriter<'a, Infallible> { - self.pages - .writer() - .skip(self.offset.value()) - .limit(self.len.value()) + pub fn writer(&'a self) -> Result, Error> { + self.inner.dma_slice.writer() } } -fn frame_range(byte_range: &Range) -> Range { - let start = byte_range.start.align_down(super::BLOCK_SIZE); - let end = byte_range.end.align_up(super::BLOCK_SIZE); - (start / super::BLOCK_SIZE)..(end / super::BLOCK_SIZE) +impl VmIo for BioSegment { + fn read(&self, offset: usize, writer: &mut VmWriter) -> Result<(), Error> { + self.inner.dma_slice.read(offset, writer) + } + + fn write(&self, offset: usize, reader: &mut VmReader) -> Result<(), Error> { + self.inner.dma_slice.write(offset, reader) + } +} + +// The timing for free the segment to the pool. +impl Drop for BioSegmentInner { + fn drop(&mut self) { + if !self.from_pool { + return; + } + if let Some(pool) = target_pool(self.direction()) { + pool.free(self); + } + } +} + +impl BioSegmentInner { + /// Returns the bio direction. + fn direction(&self) -> BioDirection { + match self.dma_slice.stream().direction() { + DmaDirection::FromDevice => BioDirection::FromDevice, + DmaDirection::ToDevice => BioDirection::ToDevice, + _ => unreachable!(), + } + } +} + +/// A pool of managing segments for block I/O requests. +/// +/// Inside the pool, it's a large chunk of `DmaStream` which +/// contains the mapped segment. The allocation/free is done by slicing +/// the `DmaStream`. +// TODO: Use a more advanced allocation algorithm to replace the naive one to improve efficiency. +struct BioSegmentPool { + pool: DmaStream, + total_blocks: usize, + direction: BioDirection, + manager: SpinLock, +} + +/// Manages the free slots in the pool. +struct PoolSlotManager { + /// A bit array to manage the occupied slots in the pool (Bit + /// value 1 represents "occupied"; 0 represents "free"). + /// The total size is currently determined by `POOL_DEFAULT_NBLOCKS`. + occupied: BitArray<[u8; POOL_DEFAULT_NBLOCKS.div_ceil(8)]>, + /// The first index of all free slots in the pool. + min_free: usize, +} + +impl BioSegmentPool { + /// Creates a new pool given the bio direction. The total number of + /// managed blocks is currently set to `POOL_DEFAULT_NBLOCKS`. + /// + /// The new pool will be allocated and mapped for later allocation. + pub fn new(direction: BioDirection) -> Self { + let total_blocks = POOL_DEFAULT_NBLOCKS; + let pool = { + let segment = FrameAllocOptions::new(total_blocks) + .uninit(true) + .alloc_contiguous() + .unwrap(); + DmaStream::map(segment, direction.into(), false).unwrap() + }; + let manager = SpinLock::new(PoolSlotManager { + occupied: BitArray::ZERO, + min_free: 0, + }); + + Self { + pool, + total_blocks, + direction, + manager, + } + } + + /// Allocates a bio segment with the given count `nblocks` + /// from the pool. + /// + /// Support two extended parameters: + /// 1. `offset_within_first_block`: the offset (in bytes) within the first block. + /// 2. `len`: the exact length (in bytes) of the wanted segment. (May + /// less than `nblocks * BLOCK_SIZE`) + /// + /// If there is no enough space in the pool, this method + /// will return `None`. + /// + /// # Panics + /// + /// If the `offset_within_first_block` exceeds the block size, or the `len` + /// exceeds the total length, this method will panic. + pub fn alloc( + &self, + nblocks: usize, + offset_within_first_block: usize, + len: usize, + ) -> Option { + assert!( + offset_within_first_block < BLOCK_SIZE + && offset_within_first_block + len <= nblocks * BLOCK_SIZE + ); + let mut manager = self.manager.lock(); + if nblocks > self.total_blocks - manager.min_free { + return None; + } + + // Find the free range + let (start, end) = { + let mut start = manager.min_free; + let mut end = start; + while end < self.total_blocks && end - start < nblocks { + if manager.occupied[end] { + start = end + 1; + end = start; + } else { + end += 1; + } + } + if end - start < nblocks { + return None; + } + (start, end) + }; + + manager.occupied[start..end].fill(true); + manager.min_free = manager.occupied[end..] + .iter() + .position(|i| !i) + .map(|pos| end + pos) + .unwrap_or(self.total_blocks); + + let dma_slice = DmaStreamSlice::new( + self.pool.clone(), + start * BLOCK_SIZE + offset_within_first_block, + len, + ); + let bio_segment = BioSegmentInner { + dma_slice, + from_pool: true, + }; + Some(bio_segment) + } + + /// Returns an allocated bio segment to the pool, + /// free the space. This method is not public and should only + /// be called automatically by `BioSegmentInner::drop()`. + /// + /// # Panics + /// + /// If the target bio segment is not allocated from the pool + /// or not the same direction, this method will panic. + fn free(&self, bio_segment: &BioSegmentInner) { + assert!(bio_segment.from_pool && bio_segment.direction() == self.direction); + let (start, end) = { + let dma_slice = &bio_segment.dma_slice; + let start = dma_slice.offset().align_down(BLOCK_SIZE) / BLOCK_SIZE; + let end = (dma_slice.offset() + dma_slice.nbytes()).align_up(BLOCK_SIZE) / BLOCK_SIZE; + + if end <= start || end > self.total_blocks { + return; + } + (start, end) + }; + + let mut manager = self.manager.lock(); + debug_assert!(manager.occupied[start..end].iter().all(|i| *i)); + manager.occupied[start..end].fill(false); + if start < manager.min_free { + manager.min_free = start; + } + } +} + +/// A pool of segments for read bio requests only. +static BIO_SEGMENT_RPOOL: Once> = Once::new(); +/// A pool of segments for write bio requests only. +static BIO_SEGMENT_WPOOL: Once> = Once::new(); +/// The default number of blocks in each pool. (16MB each for now) +const POOL_DEFAULT_NBLOCKS: usize = 4096; + +/// Initializes the bio segment pool. +pub fn bio_segment_pool_init() { + BIO_SEGMENT_RPOOL.call_once(|| Arc::new(BioSegmentPool::new(BioDirection::FromDevice))); + BIO_SEGMENT_WPOOL.call_once(|| Arc::new(BioSegmentPool::new(BioDirection::ToDevice))); +} + +/// Gets the target pool with the given `direction`. +fn target_pool(direction: BioDirection) -> Option<&'static Arc> { + match direction { + BioDirection::FromDevice => BIO_SEGMENT_RPOOL.get(), + BioDirection::ToDevice => BIO_SEGMENT_WPOOL.get(), + } +} + +impl From for DmaDirection { + fn from(direction: BioDirection) -> Self { + match direction { + BioDirection::FromDevice => DmaDirection::FromDevice, + BioDirection::ToDevice => DmaDirection::ToDevice, + } + } +} + +/// Checks if the given offset is aligned to sector. +pub fn is_sector_aligned(offset: usize) -> bool { + offset % SECTOR_SIZE == 0 } /// An aligned unsigned integer number. diff --git a/kernel/comps/block/src/impl_block_device.rs b/kernel/comps/block/src/impl_block_device.rs index c8e7bf56a..0e29cdc88 100644 --- a/kernel/comps/block/src/impl_block_device.rs +++ b/kernel/comps/block/src/impl_block_device.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: MPL-2.0 -use aster_util::segment_slice::SegmentSlice; -use ostd::mm::{ - FallibleVmRead, FallibleVmWrite, Frame, FrameAllocOptions, VmIo, VmReader, VmWriter, -}; +use ostd::mm::{VmIo, VmReader, VmWriter}; use super::{ bio::{Bio, BioEnqueueError, BioSegment, BioStatus, BioType, BioWaiter, SubmittedBio}, id::{Bid, Sid}, - BlockDevice, BLOCK_SIZE, SECTOR_SIZE, + BlockDevice, BLOCK_SIZE, +}; +use crate::{ + bio::{is_sector_aligned, BioDirection}, + prelude::*, }; -use crate::prelude::*; /// Implements several commonly used APIs for the block device to conveniently /// read and write block(s). @@ -20,9 +20,14 @@ impl dyn BlockDevice { pub fn read_blocks( &self, bid: Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { - let bio = create_bio_from_segment(BioType::Read, bid, segment); + let bio = Bio::new( + BioType::Read, + Sid::from(bid), + vec![bio_segment], + Some(general_complete_fn), + ); let status = bio.submit_and_wait(self)?; Ok(status) } @@ -31,22 +36,14 @@ impl dyn BlockDevice { pub fn read_blocks_async( &self, bid: Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { - let bio = create_bio_from_segment(BioType::Read, bid, segment); - bio.submit(self) - } - - /// Synchronously reads one block indicated by the `bid`. - pub fn read_block(&self, bid: Bid, frame: &Frame) -> Result { - let bio = create_bio_from_frame(BioType::Read, bid, frame); - let status = bio.submit_and_wait(self)?; - Ok(status) - } - - /// Asynchronously reads one block indicated by the `bid`. - pub fn read_block_async(&self, bid: Bid, frame: &Frame) -> Result { - let bio = create_bio_from_frame(BioType::Read, bid, frame); + let bio = Bio::new( + BioType::Read, + Sid::from(bid), + vec![bio_segment], + Some(general_complete_fn), + ); bio.submit(self) } @@ -54,9 +51,14 @@ impl dyn BlockDevice { pub fn write_blocks( &self, bid: Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { - let bio = create_bio_from_segment(BioType::Write, bid, segment); + let bio = Bio::new( + BioType::Write, + Sid::from(bid), + vec![bio_segment], + Some(general_complete_fn), + ); let status = bio.submit_and_wait(self)?; Ok(status) } @@ -65,22 +67,14 @@ impl dyn BlockDevice { pub fn write_blocks_async( &self, bid: Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { - let bio = create_bio_from_segment(BioType::Write, bid, segment); - bio.submit(self) - } - - /// Synchronously writes one block indicated by the `bid`. - pub fn write_block(&self, bid: Bid, frame: &Frame) -> Result { - let bio = create_bio_from_frame(BioType::Write, bid, frame); - let status = bio.submit_and_wait(self)?; - Ok(status) - } - - /// Asynchronously writes one block indicated by the `bid`. - pub fn write_block_async(&self, bid: Bid, frame: &Frame) -> Result { - let bio = create_bio_from_frame(BioType::Write, bid, frame); + let bio = Bio::new( + BioType::Write, + Sid::from(bid), + vec![bio_segment], + Some(general_complete_fn), + ); bio.submit(self) } @@ -101,7 +95,7 @@ impl VmIo for dyn BlockDevice { /// Reads consecutive bytes of several sectors in size. fn read(&self, offset: usize, writer: &mut VmWriter) -> ostd::Result<()> { let read_len = writer.avail(); - if offset % SECTOR_SIZE != 0 || read_len % SECTOR_SIZE != 0 { + if !is_sector_aligned(offset) || !is_sector_aligned(read_len) { return Err(ostd::Error::InvalidArgs); } if read_len == 0 { @@ -112,20 +106,21 @@ impl VmIo for dyn BlockDevice { let num_blocks = { let first = Bid::from_offset(offset).to_raw(); let last = Bid::from_offset(offset + read_len - 1).to_raw(); - last - first + 1 + (last - first + 1) as usize }; - let segment = FrameAllocOptions::new(num_blocks as usize) - .uninit(true) - .alloc_contiguous()?; - let bio_segment = - BioSegment::from_segment(segment.into(), offset % BLOCK_SIZE, read_len); + let bio_segment = BioSegment::alloc_inner( + num_blocks, + offset % BLOCK_SIZE, + read_len, + BioDirection::FromDevice, + ); ( Bio::new( BioType::Read, Sid::from_offset(offset), vec![bio_segment.clone()], - None, + Some(general_complete_fn), ), bio_segment, ) @@ -133,13 +128,7 @@ impl VmIo for dyn BlockDevice { let status = bio.submit_and_wait(self)?; match status { - BioStatus::Complete => { - let _ = bio_segment - .reader() - .read_fallible(writer) - .map_err(|(e, _)| e)?; - Ok(()) - } + BioStatus::Complete => bio_segment.read(0, writer), _ => Err(ostd::Error::IoError), } } @@ -147,7 +136,7 @@ impl VmIo for dyn BlockDevice { /// Writes consecutive bytes of several sectors in size. fn write(&self, offset: usize, reader: &mut VmReader) -> ostd::Result<()> { let write_len = reader.remain(); - if offset % SECTOR_SIZE != 0 || write_len % SECTOR_SIZE != 0 { + if !is_sector_aligned(offset) || !is_sector_aligned(write_len) { return Err(ostd::Error::InvalidArgs); } if write_len == 0 { @@ -158,23 +147,21 @@ impl VmIo for dyn BlockDevice { let num_blocks = { let first = Bid::from_offset(offset).to_raw(); let last = Bid::from_offset(offset + write_len - 1).to_raw(); - last - first + 1 + (last - first + 1) as usize }; - let segment = FrameAllocOptions::new(num_blocks as usize) - .uninit(true) - .alloc_contiguous()?; - segment.write(offset % BLOCK_SIZE, reader)?; - let len = segment - .writer() - .skip(offset % BLOCK_SIZE) - .write_fallible(reader) - .map_err(|(e, _)| e)?; - let bio_segment = BioSegment::from_segment(segment.into(), offset % BLOCK_SIZE, len); + let bio_segment = BioSegment::alloc_inner( + num_blocks, + offset % BLOCK_SIZE, + write_len, + BioDirection::ToDevice, + ); + bio_segment.write(0, reader)?; + Bio::new( BioType::Write, Sid::from_offset(offset), vec![bio_segment], - None, + Some(general_complete_fn), ) }; @@ -189,28 +176,27 @@ impl VmIo for dyn BlockDevice { impl dyn BlockDevice { /// Asynchronously writes consecutive bytes of several sectors in size. pub fn write_bytes_async(&self, offset: usize, buf: &[u8]) -> ostd::Result { - if offset % SECTOR_SIZE != 0 || buf.len() % SECTOR_SIZE != 0 { + let write_len = buf.len(); + if !is_sector_aligned(offset) || !is_sector_aligned(write_len) { return Err(ostd::Error::InvalidArgs); } - if buf.is_empty() { + if write_len == 0 { return Ok(BioWaiter::new()); } let bio = { let num_blocks = { let first = Bid::from_offset(offset).to_raw(); - let last = Bid::from_offset(offset + buf.len() - 1).to_raw(); - last - first + 1 + let last = Bid::from_offset(offset + write_len - 1).to_raw(); + (last - first + 1) as usize }; - let segment = FrameAllocOptions::new(num_blocks as usize) - .uninit(true) - .alloc_contiguous()?; - segment.write_bytes(offset % BLOCK_SIZE, buf)?; - let len = segment - .writer() - .skip(offset % BLOCK_SIZE) - .write(&mut buf.into()); - let bio_segment = BioSegment::from_segment(segment.into(), offset % BLOCK_SIZE, len); + let bio_segment = BioSegment::alloc_inner( + num_blocks, + offset % BLOCK_SIZE, + write_len, + BioDirection::ToDevice, + ); + bio_segment.write(0, &mut VmReader::from(buf).to_fallible())?; Bio::new( BioType::Write, Sid::from_offset(offset), @@ -224,28 +210,6 @@ impl dyn BlockDevice { } } -// TODO: Maybe we should have a builder for `Bio`. -fn create_bio_from_segment(type_: BioType, bid: Bid, segment: &SegmentSlice) -> Bio { - let bio_segment = BioSegment::from_segment(segment.clone(), 0, segment.nbytes()); - Bio::new( - type_, - Sid::from(bid), - vec![bio_segment], - Some(general_complete_fn), - ) -} - -// TODO: Maybe we should have a builder for `Bio`. -fn create_bio_from_frame(type_: BioType, bid: Bid, frame: &Frame) -> Bio { - let bio_segment = BioSegment::from_frame(frame.clone(), 0, BLOCK_SIZE); - Bio::new( - type_, - Sid::from(bid), - vec![bio_segment], - Some(general_complete_fn), - ) -} - fn general_complete_fn(bio: &SubmittedBio) { match bio.status() { BioStatus::Complete => (), diff --git a/kernel/comps/virtio/src/device/block/device.rs b/kernel/comps/virtio/src/device/block/device.rs index 0731fc1c0..ddcca3af4 100644 --- a/kernel/comps/virtio/src/device/block/device.rs +++ b/kernel/comps/virtio/src/device/block/device.rs @@ -11,7 +11,7 @@ use alloc::{ use core::{fmt::Debug, hint::spin_loop, mem::size_of}; use aster_block::{ - bio::{BioEnqueueError, BioStatus, BioType, SubmittedBio}, + bio::{bio_segment_pool_init, BioEnqueueError, BioStatus, BioType, SubmittedBio}, request_queue::{BioRequest, BioRequestSingleQueue}, BlockDeviceMeta, }; @@ -63,6 +63,8 @@ impl BlockDevice { }); aster_block::register_device(device_id, block_device); + + bio_segment_pool_init(); Ok(()) } @@ -215,11 +217,14 @@ impl DeviceInner { // Synchronize DMA mapping if read from the device if let BioType::Read = complete_request.bio_request.type_() { complete_request - .dma_bufs - .iter() - .for_each(|(stream, offset, len)| { - stream.sync(*offset..*offset + *len).unwrap(); - }); + .bio_request + .bios() + .flat_map(|bio| { + bio.segments() + .iter() + .map(|segment| segment.inner_dma_slice()) + }) + .for_each(|dma_slice| dma_slice.sync().unwrap()); } // Completes the bio request @@ -301,11 +306,10 @@ impl DeviceInner { /// Reads data from the device, this function is non-blocking. fn read(&self, bio_request: BioRequest) { - let dma_streams = Self::dma_stream_map(&bio_request); - let id = self.id_allocator.disable_irq().lock().alloc().unwrap(); let req_slice = { - let req_slice = DmaStreamSlice::new(&self.block_requests, id * REQ_SIZE, REQ_SIZE); + let req_slice = + DmaStreamSlice::new(self.block_requests.clone(), id * REQ_SIZE, REQ_SIZE); let req = BlockReq { type_: ReqType::In as _, reserved: 0, @@ -317,18 +321,21 @@ impl DeviceInner { }; let resp_slice = { - let resp_slice = DmaStreamSlice::new(&self.block_responses, id * RESP_SIZE, RESP_SIZE); + let resp_slice = + DmaStreamSlice::new(self.block_responses.clone(), id * RESP_SIZE, RESP_SIZE); resp_slice.write_val(0, &BlockResp::default()).unwrap(); resp_slice }; - let dma_slices: Vec> = dma_streams - .iter() - .map(|(stream, offset, len)| DmaStreamSlice::new(stream, *offset, *len)) - .collect(); let outputs = { - let mut outputs: Vec<&DmaStreamSlice<_>> = Vec::with_capacity(dma_streams.len() + 1); - outputs.extend(dma_slices.iter()); + let mut outputs: Vec<&DmaStreamSlice<_>> = + Vec::with_capacity(bio_request.num_segments() + 1); + let dma_slices_iter = bio_request.bios().flat_map(|bio| { + bio.segments() + .iter() + .map(|segment| segment.inner_dma_slice()) + }); + outputs.extend(dma_slices_iter); outputs.push(&resp_slice); outputs }; @@ -352,7 +359,7 @@ impl DeviceInner { } // Records the submitted request - let submitted_request = SubmittedRequest::new(id as u16, bio_request, dma_streams); + let submitted_request = SubmittedRequest::new(id as u16, bio_request); self.submitted_requests .disable_irq() .lock() @@ -363,11 +370,10 @@ impl DeviceInner { /// Writes data to the device, this function is non-blocking. fn write(&self, bio_request: BioRequest) { - let dma_streams = Self::dma_stream_map(&bio_request); - let id = self.id_allocator.disable_irq().lock().alloc().unwrap(); let req_slice = { - let req_slice = DmaStreamSlice::new(&self.block_requests, id * REQ_SIZE, REQ_SIZE); + let req_slice = + DmaStreamSlice::new(self.block_requests.clone(), id * REQ_SIZE, REQ_SIZE); let req = BlockReq { type_: ReqType::Out as _, reserved: 0, @@ -379,19 +385,22 @@ impl DeviceInner { }; let resp_slice = { - let resp_slice = DmaStreamSlice::new(&self.block_responses, id * RESP_SIZE, RESP_SIZE); + let resp_slice = + DmaStreamSlice::new(self.block_responses.clone(), id * RESP_SIZE, RESP_SIZE); resp_slice.write_val(0, &BlockResp::default()).unwrap(); resp_slice }; - let dma_slices: Vec> = dma_streams - .iter() - .map(|(stream, offset, len)| DmaStreamSlice::new(stream, *offset, *len)) - .collect(); let inputs = { - let mut inputs: Vec<&DmaStreamSlice<_>> = Vec::with_capacity(dma_streams.len() + 1); + let mut inputs: Vec<&DmaStreamSlice<_>> = + Vec::with_capacity(bio_request.num_segments() + 1); inputs.push(&req_slice); - inputs.extend(dma_slices.iter()); + let dma_slices_iter = bio_request.bios().flat_map(|bio| { + bio.segments() + .iter() + .map(|segment| segment.inner_dma_slice()) + }); + inputs.extend(dma_slices_iter); inputs }; @@ -413,7 +422,7 @@ impl DeviceInner { } // Records the submitted request - let submitted_request = SubmittedRequest::new(id as u16, bio_request, dma_streams); + let submitted_request = SubmittedRequest::new(id as u16, bio_request); self.submitted_requests .disable_irq() .lock() @@ -465,7 +474,7 @@ impl DeviceInner { } // Records the submitted request - let submitted_request = SubmittedRequest::new(id as u16, bio_request, Vec::new()); + let submitted_request = SubmittedRequest::new(id as u16, bio_request); self.submitted_requests .disable_irq() .lock() @@ -473,27 +482,6 @@ impl DeviceInner { return; } } - - /// Performs DMA mapping for the segments in bio request. - fn dma_stream_map(bio_request: &BioRequest) -> Vec<(DmaStream, usize, usize)> { - let dma_direction = match bio_request.type_() { - BioType::Read => DmaDirection::FromDevice, - BioType::Write => DmaDirection::ToDevice, - _ => todo!(), - }; - - bio_request - .bios() - .flat_map(|bio| { - bio.segments().iter().map(|segment| { - let dma_stream = - DmaStream::map(segment.pages().clone().into(), dma_direction, false) - .unwrap(); - (dma_stream, segment.offset(), segment.nbytes()) - }) - }) - .collect() - } } /// A submitted bio request for callback. @@ -501,16 +489,11 @@ impl DeviceInner { struct SubmittedRequest { id: u16, bio_request: BioRequest, - dma_bufs: Vec<(DmaStream, usize, usize)>, } impl SubmittedRequest { - pub fn new(id: u16, bio_request: BioRequest, dma_bufs: Vec<(DmaStream, usize, usize)>) -> Self { - Self { - id, - bio_request, - dma_bufs, - } + pub fn new(id: u16, bio_request: BioRequest) -> Self { + Self { id, bio_request } } } diff --git a/kernel/src/fs/exfat/fs.rs b/kernel/src/fs/exfat/fs.rs index 3b9a8a34a..3af146538 100644 --- a/kernel/src/fs/exfat/fs.rs +++ b/kernel/src/fs/exfat/fs.rs @@ -5,7 +5,11 @@ use core::{num::NonZeroUsize, ops::Range, sync::atomic::AtomicU64}; -use aster_block::{bio::BioWaiter, id::BlockId, BlockDevice}; +use aster_block::{ + bio::{BioDirection, BioSegment, BioWaiter}, + id::BlockId, + BlockDevice, +}; use hashbrown::HashMap; use lru::LruCache; use ostd::mm::Frame; @@ -368,9 +372,11 @@ impl PageCacheBackend for ExfatFS { if self.fs_size() < idx * PAGE_SIZE { return_errno_with_message!(Errno::EINVAL, "invalid read size") } + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::FromDevice); let waiter = self .block_device - .read_block_async(BlockId::new(idx as u64), frame)?; + .read_blocks_async(BlockId::new(idx as u64), bio_segment)?; Ok(waiter) } @@ -378,9 +384,11 @@ impl PageCacheBackend for ExfatFS { if self.fs_size() < idx * PAGE_SIZE { return_errno_with_message!(Errno::EINVAL, "invalid write size") } + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::ToDevice); let waiter = self .block_device - .write_block_async(BlockId::new(idx as u64), frame)?; + .write_blocks_async(BlockId::new(idx as u64), bio_segment)?; Ok(waiter) } diff --git a/kernel/src/fs/exfat/inode.rs b/kernel/src/fs/exfat/inode.rs index 1e6604189..a01f094ec 100644 --- a/kernel/src/fs/exfat/inode.rs +++ b/kernel/src/fs/exfat/inode.rs @@ -8,12 +8,12 @@ use core::{cmp::Ordering, time::Duration}; pub(super) use align_ext::AlignExt; use aster_block::{ - bio::BioWaiter, + bio::{BioDirection, BioSegment, BioWaiter}, id::{Bid, BlockId}, BLOCK_SIZE, }; use aster_rights::Full; -use ostd::mm::{Frame, FrameAllocOptions, VmIo}; +use ostd::mm::{Frame, VmIo}; use super::{ constants::*, @@ -141,9 +141,11 @@ impl PageCacheBackend for ExfatInode { return_errno_with_message!(Errno::EINVAL, "Invalid read size") } let sector_id = inner.get_sector_id(idx * PAGE_SIZE / inner.fs().sector_size())?; - let waiter = inner.fs().block_device().read_block_async( + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::FromDevice); + let waiter = inner.fs().block_device().read_blocks_async( BlockId::from_offset(sector_id * inner.fs().sector_size()), - frame, + bio_segment, )?; Ok(waiter) } @@ -156,9 +158,11 @@ impl PageCacheBackend for ExfatInode { // FIXME: We may need to truncate the file if write_page fails. // To fix this issue, we need to change the interface of the PageCacheBackend trait. - let waiter = inner.fs().block_device().write_block_async( + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::ToDevice); + let waiter = inner.fs().block_device().write_blocks_async( BlockId::from_offset(sector_id * inner.fs().sector_size()), - frame, + bio_segment, )?; Ok(waiter) } @@ -1263,10 +1267,7 @@ impl Inode for ExfatInode { .discard_range(read_off..read_off + read_len); let mut buf_offset = 0; - let frame = FrameAllocOptions::new(1) - .uninit(true) - .alloc_single() - .unwrap(); + let bio_segment = BioSegment::alloc(1, BioDirection::FromDevice); let start_pos = inner.start_chain.walk_to_cluster_at_offset(read_off)?; let cluster_size = inner.fs().cluster_size(); @@ -1275,8 +1276,11 @@ impl Inode for ExfatInode { for _ in Bid::from_offset(read_off)..Bid::from_offset(read_off + read_len) { let physical_bid = Bid::from_offset(cur_cluster.cluster_id() as usize * cluster_size + cur_offset); - inner.fs().block_device().read_block(physical_bid, &frame)?; - frame.read(0, writer).unwrap(); + inner + .fs() + .block_device() + .read_blocks(physical_bid, bio_segment.clone())?; + bio_segment.reader().unwrap().read_fallible(writer)?; buf_offset += BLOCK_SIZE; cur_offset += BLOCK_SIZE; @@ -1370,23 +1374,18 @@ impl Inode for ExfatInode { let inner = self.inner.upread(); + let bio_segment = BioSegment::alloc(1, BioDirection::ToDevice); let start_pos = inner.start_chain.walk_to_cluster_at_offset(offset)?; let cluster_size = inner.fs().cluster_size(); let mut cur_cluster = start_pos.0.clone(); let mut cur_offset = start_pos.1; for _ in Bid::from_offset(offset)..Bid::from_offset(end_offset) { - let frame = { - let frame = FrameAllocOptions::new(1) - .uninit(true) - .alloc_single() - .unwrap(); - frame.write(0, reader)?; - frame - }; + bio_segment.writer().unwrap().write_fallible(reader)?; let physical_bid = Bid::from_offset(cur_cluster.cluster_id() as usize * cluster_size + cur_offset); let fs = inner.fs(); - fs.block_device().write_block(physical_bid, &frame)?; + fs.block_device() + .write_blocks(physical_bid, bio_segment.clone())?; cur_offset += BLOCK_SIZE; if cur_offset >= cluster_size { diff --git a/kernel/src/fs/exfat/mod.rs b/kernel/src/fs/exfat/mod.rs index 455369f92..6440b85b9 100644 --- a/kernel/src/fs/exfat/mod.rs +++ b/kernel/src/fs/exfat/mod.rs @@ -83,6 +83,7 @@ mod test { for seg in bio.segments() { let size = match bio.type_() { BioType::Read => seg + .inner_segment() .writer() .write(&mut self.queue.0.reader().skip(cur_device_ofs)), BioType::Write => self @@ -90,7 +91,7 @@ mod test { .0 .writer() .skip(cur_device_ofs) - .write(&mut seg.reader()), + .write(&mut seg.inner_segment().reader()), _ => 0, }; cur_device_ofs += size; diff --git a/kernel/src/fs/ext2/block_group.rs b/kernel/src/fs/ext2/block_group.rs index c076060ea..8391967e8 100644 --- a/kernel/src/fs/ext2/block_group.rs +++ b/kernel/src/fs/ext2/block_group.rs @@ -28,7 +28,7 @@ struct BlockGroupImpl { impl BlockGroup { /// Loads and constructs a block group. pub fn load( - group_descriptors_segment: &SegmentSlice, + group_descriptors_segment: &Segment, idx: usize, block_device: &dyn BlockDevice, super_block: &SuperBlock, @@ -320,12 +320,22 @@ impl Debug for BlockGroup { impl PageCacheBackend for BlockGroupImpl { fn read_page_async(&self, idx: usize, frame: &Frame) -> Result { let bid = self.inode_table_bid + idx as Ext2Bid; - self.fs.upgrade().unwrap().read_block_async(bid, frame) + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::FromDevice); + self.fs + .upgrade() + .unwrap() + .read_blocks_async(bid, bio_segment) } fn write_page_async(&self, idx: usize, frame: &Frame) -> Result { let bid = self.inode_table_bid + idx as Ext2Bid; - self.fs.upgrade().unwrap().write_block_async(bid, frame) + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::ToDevice); + self.fs + .upgrade() + .unwrap() + .write_blocks_async(bid, bio_segment) } fn npages(&self) -> usize { diff --git a/kernel/src/fs/ext2/fs.rs b/kernel/src/fs/ext2/fs.rs index 1d0f573c9..5026beaf5 100644 --- a/kernel/src/fs/ext2/fs.rs +++ b/kernel/src/fs/ext2/fs.rs @@ -23,7 +23,7 @@ pub struct Ext2 { blocks_per_group: Ext2Bid, inode_size: usize, block_size: usize, - group_descriptors_segment: SegmentSlice, + group_descriptors_segment: Segment, self_ref: Weak, } @@ -48,9 +48,10 @@ impl Ext2 { .div_ceil(BLOCK_SIZE); let segment = FrameAllocOptions::new(npages) .uninit(true) - .alloc_contiguous()? - .into(); - match block_device.read_blocks(super_block.group_descriptors_bid(0), &segment)? { + .alloc_contiguous()?; + let bio_segment = + BioSegment::new_from_segment(segment.clone(), BioDirection::FromDevice); + match block_device.read_blocks(super_block.group_descriptors_bid(0), bio_segment)? { BioStatus::Complete => (), err_status => { return Err(Error::from(err_status)); @@ -62,7 +63,7 @@ impl Ext2 { // Load the block groups information let load_block_groups = |fs: Weak, block_device: &dyn BlockDevice, - group_descriptors_segment: &SegmentSlice| + group_descriptors_segment: &Segment| -> Result> { let block_groups_count = super_block.block_groups_count() as usize; let mut block_groups = Vec::with_capacity(block_groups_count); @@ -298,10 +299,10 @@ impl Ext2 { } /// Reads contiguous blocks starting from the `bid` synchronously. - pub(super) fn read_blocks(&self, bid: Ext2Bid, segment: &SegmentSlice) -> Result<()> { + pub(super) fn read_blocks(&self, bid: Ext2Bid, bio_segment: BioSegment) -> Result<()> { let status = self .block_device - .read_blocks(Bid::new(bid as u64), segment)?; + .read_blocks(Bid::new(bid as u64), bio_segment)?; match status { BioStatus::Complete => Ok(()), err_status => Err(Error::from(err_status)), @@ -312,36 +313,19 @@ impl Ext2 { pub(super) fn read_blocks_async( &self, bid: Ext2Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { let waiter = self .block_device - .read_blocks_async(Bid::new(bid as u64), segment)?; - Ok(waiter) - } - - /// Reads one block indicated by the `bid` synchronously. - pub(super) fn read_block(&self, bid: Ext2Bid, frame: &Frame) -> Result<()> { - let status = self.block_device.read_block(Bid::new(bid as u64), frame)?; - match status { - BioStatus::Complete => Ok(()), - err_status => Err(Error::from(err_status)), - } - } - - /// Reads one block indicated by the `bid` asynchronously. - pub(super) fn read_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { - let waiter = self - .block_device - .read_block_async(Bid::new(bid as u64), frame)?; + .read_blocks_async(Bid::new(bid as u64), bio_segment)?; Ok(waiter) } /// Writes contiguous blocks starting from the `bid` synchronously. - pub(super) fn write_blocks(&self, bid: Ext2Bid, segment: &SegmentSlice) -> Result<()> { + pub(super) fn write_blocks(&self, bid: Ext2Bid, bio_segment: BioSegment) -> Result<()> { let status = self .block_device - .write_blocks(Bid::new(bid as u64), segment)?; + .write_blocks(Bid::new(bid as u64), bio_segment)?; match status { BioStatus::Complete => Ok(()), err_status => Err(Error::from(err_status)), @@ -352,28 +336,11 @@ impl Ext2 { pub(super) fn write_blocks_async( &self, bid: Ext2Bid, - segment: &SegmentSlice, + bio_segment: BioSegment, ) -> Result { let waiter = self .block_device - .write_blocks_async(Bid::new(bid as u64), segment)?; - Ok(waiter) - } - - /// Writes one block indicated by the `bid` synchronously. - pub(super) fn write_block(&self, bid: Ext2Bid, frame: &Frame) -> Result<()> { - let status = self.block_device.write_block(Bid::new(bid as u64), frame)?; - match status { - BioStatus::Complete => Ok(()), - err_status => Err(Error::from(err_status)), - } - } - - /// Writes one block indicated by the `bid` asynchronously. - pub(super) fn write_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { - let waiter = self - .block_device - .write_block_async(Bid::new(bid as u64), frame)?; + .write_blocks_async(Bid::new(bid as u64), bio_segment)?; Ok(waiter) } @@ -397,9 +364,13 @@ impl Ext2 { self.block_device .write_bytes_async(SUPER_BLOCK_OFFSET, raw_super_block.as_bytes())?, ); + let group_descriptors_bio_segment = BioSegment::new_from_segment( + self.group_descriptors_segment.clone(), + BioDirection::ToDevice, + ); bio_waiter.concat(self.block_device.write_blocks_async( super_block.group_descriptors_bid(0), - &self.group_descriptors_segment, + group_descriptors_bio_segment.clone(), )?); bio_waiter .wait() @@ -418,7 +389,7 @@ impl Ext2 { )?); bio_waiter.concat(self.block_device.write_blocks_async( super_block.group_descriptors_bid(idx as usize), - &self.group_descriptors_segment, + group_descriptors_bio_segment.clone(), )?); bio_waiter.wait().ok_or_else(|| { Error::with_message(Errno::EIO, "failed to sync backup metadata") diff --git a/kernel/src/fs/ext2/indirect_block_cache.rs b/kernel/src/fs/ext2/indirect_block_cache.rs index a9cd7de9b..399ee15ce 100644 --- a/kernel/src/fs/ext2/indirect_block_cache.rs +++ b/kernel/src/fs/ext2/indirect_block_cache.rs @@ -42,7 +42,9 @@ impl IndirectBlockCache { let fs = self.fs(); let load_block = || -> Result { let mut block = IndirectBlock::alloc_uninit()?; - fs.read_block(bid, &block.frame)?; + let bio_segment = + BioSegment::new_from_segment(block.frame.clone().into(), BioDirection::FromDevice); + fs.read_blocks(bid, bio_segment)?; block.state = State::UpToDate; Ok(block) }; @@ -59,7 +61,9 @@ impl IndirectBlockCache { let fs = self.fs(); let load_block = || -> Result { let mut block = IndirectBlock::alloc_uninit()?; - fs.read_block(bid, &block.frame)?; + let bio_segment = + BioSegment::new_from_segment(block.frame.clone().into(), BioDirection::FromDevice); + fs.read_blocks(bid, bio_segment)?; block.state = State::UpToDate; Ok(block) }; @@ -104,7 +108,11 @@ impl IndirectBlockCache { for _ in 0..num { let (bid, block) = self.cache.pop_lru().unwrap(); if block.is_dirty() { - bio_waiter.concat(self.fs().write_block_async(bid, &block.frame)?); + let bio_segment = BioSegment::new_from_segment( + block.frame.clone().into(), + BioDirection::ToDevice, + ); + bio_waiter.concat(self.fs().write_blocks_async(bid, bio_segment)?); } } diff --git a/kernel/src/fs/ext2/inode.rs b/kernel/src/fs/ext2/inode.rs index 059d8900a..d62322c8f 100644 --- a/kernel/src/fs/ext2/inode.rs +++ b/kernel/src/fs/ext2/inode.rs @@ -920,6 +920,7 @@ impl Inner { } pub fn read_direct_at(&self, offset: usize, writer: &mut VmWriter) -> Result { + debug_assert!(is_block_aligned(offset) && is_block_aligned(writer.avail())); let (offset, read_len) = { let file_size = self.inode_impl.file_size(); let start = file_size.min(offset).align_down(BLOCK_SIZE); @@ -928,17 +929,16 @@ impl Inner { .align_down(BLOCK_SIZE); (start, end - start) }; + if read_len == 0 { + return Ok(read_len); + } self.page_cache.discard_range(offset..offset + read_len); let start_bid = Bid::from_offset(offset).to_raw() as Ext2Bid; let buf_nblocks = read_len / BLOCK_SIZE; - let segment = FrameAllocOptions::new(buf_nblocks) - .uninit(true) - .alloc_contiguous()? - .into(); + self.inode_impl + .read_blocks(start_bid, buf_nblocks, writer)?; - self.inode_impl.read_blocks(start_bid, &segment)?; - segment.read(0, writer)?; Ok(read_len) } @@ -958,6 +958,7 @@ impl Inner { } pub fn write_direct_at(&mut self, offset: usize, reader: &mut VmReader) -> Result { + debug_assert!(is_block_aligned(offset) && is_block_aligned(reader.remain())); let file_size = self.inode_impl.file_size(); let write_len = reader.remain(); let end_offset = offset + write_len; @@ -972,15 +973,9 @@ impl Inner { let start_bid = Bid::from_offset(offset).to_raw() as Ext2Bid; let buf_nblocks = write_len / BLOCK_SIZE; - let segment = { - let segment = FrameAllocOptions::new(buf_nblocks) - .uninit(true) - .alloc_contiguous()?; - segment.write(0, reader)?; - segment.into() - }; + self.inode_impl + .write_blocks(start_bid, buf_nblocks, reader)?; - self.inode_impl.write_blocks(start_bid, &segment)?; Ok(write_len) } @@ -1114,58 +1109,94 @@ impl InodeImpl_ { self.inode().fs() } - pub fn read_blocks_async(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result { - let nblocks = blocks.nframes(); + pub fn read_blocks_async( + &self, + bid: Ext2Bid, + nblocks: usize, + writer: &mut VmWriter, + ) -> Result { + debug_assert!(nblocks * BLOCK_SIZE <= writer.avail()); let mut bio_waiter = BioWaiter::new(); - let mut blocks_offset = 0; for dev_range in DeviceRangeReader::new(self, bid..bid + nblocks as Ext2Bid)? { - let first_bid = dev_range.start as Ext2Bid; - let range_len = dev_range.len(); - let segment = blocks.range(blocks_offset..blocks_offset + range_len); + let start_bid = dev_range.start as Ext2Bid; + let range_nblocks = dev_range.len(); - let waiter = self.fs().read_blocks_async(first_bid, &segment)?; + let bio_segment = BioSegment::alloc(range_nblocks, BioDirection::FromDevice); + bio_segment.reader().unwrap().read_fallible(writer)?; + + let waiter = self.fs().read_blocks_async(start_bid, bio_segment)?; bio_waiter.concat(waiter); - - blocks_offset += range_len; } Ok(bio_waiter) } - pub fn read_blocks(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result<()> { - match self.read_blocks_async(bid, blocks)?.wait() { + pub fn read_blocks(&self, bid: Ext2Bid, nblocks: usize, writer: &mut VmWriter) -> Result<()> { + match self.read_blocks_async(bid, nblocks, writer)?.wait() { Some(BioStatus::Complete) => Ok(()), _ => return_errno!(Errno::EIO), } } - pub fn write_blocks_async(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result { - let nblocks = blocks.nframes(); + pub fn read_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { let mut bio_waiter = BioWaiter::new(); - let mut blocks_offset = 0; - for dev_range in DeviceRangeReader::new(self, bid..bid + nblocks as Ext2Bid)? { - let first_bid = dev_range.start as Ext2Bid; - let range_len = dev_range.len(); - let segment = blocks.range(blocks_offset..blocks_offset + range_len); - - let waiter = self.fs().write_blocks_async(first_bid, &segment)?; + for dev_range in DeviceRangeReader::new(self, bid..bid + 1 as Ext2Bid)? { + let start_bid = dev_range.start as Ext2Bid; + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::FromDevice); + let waiter = self.fs().read_blocks_async(start_bid, bio_segment)?; bio_waiter.concat(waiter); - - blocks_offset += range_len; } Ok(bio_waiter) } - pub fn write_blocks(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result<()> { - match self.write_blocks_async(bid, blocks)?.wait() { + pub fn write_blocks_async( + &self, + bid: Ext2Bid, + nblocks: usize, + reader: &mut VmReader, + ) -> Result { + debug_assert_eq!(nblocks * BLOCK_SIZE, reader.remain()); + let mut bio_waiter = BioWaiter::new(); + + for dev_range in DeviceRangeReader::new(self, bid..bid + nblocks as Ext2Bid)? { + let start_bid = dev_range.start as Ext2Bid; + let range_nblocks = dev_range.len(); + + let bio_segment = BioSegment::alloc(range_nblocks, BioDirection::ToDevice); + bio_segment.writer().unwrap().write_fallible(reader)?; + + let waiter = self.fs().write_blocks_async(start_bid, bio_segment)?; + bio_waiter.concat(waiter); + } + + Ok(bio_waiter) + } + + pub fn write_blocks(&self, bid: Ext2Bid, nblocks: usize, reader: &mut VmReader) -> Result<()> { + match self.write_blocks_async(bid, nblocks, reader)?.wait() { Some(BioStatus::Complete) => Ok(()), _ => return_errno!(Errno::EIO), } } + pub fn write_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { + let mut bio_waiter = BioWaiter::new(); + + for dev_range in DeviceRangeReader::new(self, bid..bid + 1 as Ext2Bid)? { + let start_bid = dev_range.start as Ext2Bid; + let bio_segment = + BioSegment::new_from_segment(frame.clone().into(), BioDirection::ToDevice); + let waiter = self.fs().write_blocks_async(start_bid, bio_segment)?; + bio_waiter.concat(waiter); + } + + Ok(bio_waiter) + } + pub fn resize(&mut self, new_size: usize) -> Result<()> { let old_size = self.desc.size; if new_size > old_size { @@ -1816,21 +1847,39 @@ impl InodeImpl { } /// Reads one or multiple blocks to the segment start from `bid` asynchronously. - pub fn read_blocks_async(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result { - self.0.read().read_blocks_async(bid, blocks) + pub fn read_blocks_async( + &self, + bid: Ext2Bid, + nblocks: usize, + writer: &mut VmWriter, + ) -> Result { + self.0.read().read_blocks_async(bid, nblocks, writer) } - pub fn read_blocks(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result<()> { - self.0.read().read_blocks(bid, blocks) + pub fn read_blocks(&self, bid: Ext2Bid, nblocks: usize, writer: &mut VmWriter) -> Result<()> { + self.0.read().read_blocks(bid, nblocks, writer) + } + + pub fn read_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { + self.0.read().read_block_async(bid, frame) } /// Writes one or multiple blocks from the segment start from `bid` asynchronously. - pub fn write_blocks_async(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result { - self.0.read().write_blocks_async(bid, blocks) + pub fn write_blocks_async( + &self, + bid: Ext2Bid, + nblocks: usize, + reader: &mut VmReader, + ) -> Result { + self.0.read().write_blocks_async(bid, nblocks, reader) } - pub fn write_blocks(&self, bid: Ext2Bid, blocks: &SegmentSlice) -> Result<()> { - self.0.read().write_blocks(bid, blocks) + pub fn write_blocks(&self, bid: Ext2Bid, nblocks: usize, reader: &mut VmReader) -> Result<()> { + self.0.read().write_blocks(bid, nblocks, reader) + } + + pub fn write_block_async(&self, bid: Ext2Bid, frame: &Frame) -> Result { + self.0.read().write_block_async(bid, frame) } pub fn set_device_id(&self, device_id: u64) { @@ -1894,12 +1943,12 @@ impl InodeImpl { impl PageCacheBackend for InodeImpl { fn read_page_async(&self, idx: usize, frame: &Frame) -> Result { let bid = idx as Ext2Bid; - self.read_blocks_async(bid, &SegmentSlice::from(frame.clone())) + self.read_block_async(bid, frame) } fn write_page_async(&self, idx: usize, frame: &Frame) -> Result { let bid = idx as Ext2Bid; - self.write_blocks_async(bid, &SegmentSlice::from(frame.clone())) + self.write_block_async(bid, frame) } fn npages(&self) -> usize { diff --git a/kernel/src/fs/ext2/prelude.rs b/kernel/src/fs/ext2/prelude.rs index eaf376002..015bae3de 100644 --- a/kernel/src/fs/ext2/prelude.rs +++ b/kernel/src/fs/ext2/prelude.rs @@ -7,14 +7,13 @@ pub(super) use core::{ pub(super) use align_ext::AlignExt; pub(super) use aster_block::{ - bio::{BioStatus, BioWaiter}, + bio::{BioDirection, BioSegment, BioStatus, BioWaiter}, id::Bid, BlockDevice, BLOCK_SIZE, }; pub(super) use aster_rights::Full; -pub(super) use aster_util::segment_slice::SegmentSlice; pub(super) use ostd::{ - mm::{Frame, FrameAllocOptions, VmIo}, + mm::{Frame, FrameAllocOptions, Segment, VmIo}, sync::{RwMutex, RwMutexReadGuard, RwMutexWriteGuard}, }; pub(super) use static_assertions::const_assert;