Refactor virtio drivers with DMA APIs

This commit is contained in:
Jianfeng Jiang
2024-03-12 11:01:50 +00:00
committed by Tate, Hongliang Tian
parent 5e127b2da0
commit cd1575bc6d
22 changed files with 853 additions and 311 deletions

View File

@ -37,4 +37,3 @@ smoltcp = { version = "0.9.1", default-features = false, features = [
"socket-raw",
"socket-dhcpv4",
] }
[features]

View File

@ -99,18 +99,12 @@ impl DeviceInner {
let queue = VirtQueue::new(0, Self::QUEUE_SIZE, transport.as_mut())
.expect("create virtqueue failed");
let block_requests = {
let vm_segment = VmAllocOptions::new(1)
.is_contiguous(true)
.alloc_contiguous()
.unwrap();
let vm_segment = VmAllocOptions::new(1).alloc_contiguous().unwrap();
DmaStream::map(vm_segment, DmaDirection::Bidirectional, false).unwrap()
};
assert!(Self::QUEUE_SIZE as usize * REQ_SIZE <= block_requests.nbytes());
let block_responses = {
let vm_segment = VmAllocOptions::new(1)
.is_contiguous(true)
.alloc_contiguous()
.unwrap();
let vm_segment = VmAllocOptions::new(1).alloc_contiguous().unwrap();
DmaStream::map(vm_segment, DmaDirection::Bidirectional, false).unwrap()
};
assert!(Self::QUEUE_SIZE as usize * RESP_SIZE <= block_responses.nbytes());
@ -222,7 +216,6 @@ impl DeviceInner {
const MAX_ID_LENGTH: usize = 20;
let device_id_stream = {
let segment = VmAllocOptions::new(1)
.is_contiguous(true)
.uninit(true)
.alloc_contiguous()
.unwrap();

View File

@ -4,7 +4,12 @@ use alloc::{boxed::Box, fmt::Debug, string::ToString, sync::Arc, vec::Vec};
use core::hint::spin_loop;
use aster_console::{AnyConsoleDevice, ConsoleCallback};
use aster_frame::{io_mem::IoMem, sync::SpinLock, trap::TrapFrame, vm::PAGE_SIZE};
use aster_frame::{
io_mem::IoMem,
sync::SpinLock,
trap::TrapFrame,
vm::{DmaDirection, DmaStream, DmaStreamSlice, VmAllocOptions, VmReader},
};
use aster_util::safe_ptr::SafePtr;
use log::debug;
@ -17,62 +22,39 @@ use crate::{
pub struct ConsoleDevice {
config: SafePtr<VirtioConsoleConfig, IoMem>,
transport: Box<dyn VirtioTransport>,
transport: SpinLock<Box<dyn VirtioTransport>>,
receive_queue: SpinLock<VirtQueue>,
transmit_queue: SpinLock<VirtQueue>,
buffer: SpinLock<Box<[u8; PAGE_SIZE]>>,
send_buffer: DmaStream,
receive_buffer: DmaStream,
callbacks: SpinLock<Vec<&'static ConsoleCallback>>,
}
impl AnyConsoleDevice for ConsoleDevice {
fn send(&self, value: &[u8]) {
let mut transmit_queue = self.transmit_queue.lock_irq_disabled();
transmit_queue.add_buf(&[value], &[]).unwrap();
if transmit_queue.should_notify() {
transmit_queue.notify();
let mut reader = VmReader::from(value);
while reader.remain() > 0 {
let mut writer = self.send_buffer.writer().unwrap();
let len = writer.write(&mut reader);
self.send_buffer.sync(0..len).unwrap();
let slice = DmaStreamSlice::new(&self.send_buffer, 0, len);
transmit_queue.add_dma_buf(&[&slice], &[]).unwrap();
if transmit_queue.should_notify() {
transmit_queue.notify();
}
while !transmit_queue.can_pop() {
spin_loop();
}
transmit_queue.pop_used().unwrap();
}
while !transmit_queue.can_pop() {
spin_loop();
}
transmit_queue.pop_used().unwrap();
}
fn recv(&self, buf: &mut [u8]) -> Option<usize> {
let mut receive_queue = self.receive_queue.lock_irq_disabled();
if !receive_queue.can_pop() {
return None;
}
let (_, len) = receive_queue.pop_used().unwrap();
let mut recv_buffer = self.buffer.lock();
buf.copy_from_slice(&recv_buffer.as_ref()[..len as usize]);
receive_queue.add_buf(&[], &[recv_buffer.as_mut()]).unwrap();
if receive_queue.should_notify() {
receive_queue.notify();
}
Some(len as usize)
}
fn register_callback(&self, callback: &'static (dyn Fn(&[u8]) + Send + Sync)) {
self.callbacks.lock().push(callback);
}
fn handle_irq(&self) {
let mut receive_queue = self.receive_queue.lock_irq_disabled();
if !receive_queue.can_pop() {
return;
}
let (_, len) = receive_queue.pop_used().unwrap();
let mut recv_buffer = self.buffer.lock();
let buffer = &recv_buffer.as_ref()[..len as usize];
let lock = self.callbacks.lock();
for callback in lock.iter() {
callback.call((buffer,));
}
receive_queue.add_buf(&[], &[recv_buffer.as_mut()]).unwrap();
if receive_queue.should_notify() {
receive_queue.notify();
}
fn register_callback(&self, callback: &'static ConsoleCallback) {
self.callbacks.lock_irq_disabled().push(callback);
}
}
@ -104,41 +86,76 @@ impl ConsoleDevice {
let transmit_queue =
SpinLock::new(VirtQueue::new(TRANSMIT0_QUEUE_INDEX, 2, transport.as_mut()).unwrap());
let mut device = Self {
config,
transport,
receive_queue,
transmit_queue,
buffer: SpinLock::new(Box::new([0; PAGE_SIZE])),
callbacks: SpinLock::new(Vec::new()),
let send_buffer = {
let vm_segment = VmAllocOptions::new(1).alloc_contiguous().unwrap();
DmaStream::map(vm_segment, DmaDirection::ToDevice, false).unwrap()
};
let mut receive_queue = device.receive_queue.lock();
let receive_buffer = {
let vm_segment = VmAllocOptions::new(1).alloc_contiguous().unwrap();
DmaStream::map(vm_segment, DmaDirection::FromDevice, false).unwrap()
};
let device = Arc::new(Self {
config,
transport: SpinLock::new(transport),
receive_queue,
transmit_queue,
send_buffer,
receive_buffer,
callbacks: SpinLock::new(Vec::new()),
});
let mut receive_queue = device.receive_queue.lock_irq_disabled();
receive_queue
.add_buf(&[], &[device.buffer.lock().as_mut()])
.add_dma_buf(&[], &[&device.receive_buffer])
.unwrap();
if receive_queue.should_notify() {
receive_queue.notify();
}
drop(receive_queue);
device
.transport
// Register irq callbacks
let mut transport = device.transport.lock_irq_disabled();
let handle_console_input = {
let device = device.clone();
move |_: &TrapFrame| device.handle_recv_irq()
};
transport
.register_queue_callback(RECV0_QUEUE_INDEX, Box::new(handle_console_input), false)
.unwrap();
device
.transport
transport
.register_cfg_callback(Box::new(config_space_change))
.unwrap();
device.transport.finish_init();
transport.finish_init();
drop(transport);
aster_console::register_device(DEVICE_NAME.to_string(), Arc::new(device));
aster_console::register_device(DEVICE_NAME.to_string(), device);
Ok(())
}
}
fn handle_console_input(_: &TrapFrame) {
aster_console::get_device(DEVICE_NAME).unwrap().handle_irq();
fn handle_recv_irq(&self) {
let mut receive_queue = self.receive_queue.lock_irq_disabled();
if !receive_queue.can_pop() {
return;
}
let (_, len) = receive_queue.pop_used().unwrap();
self.receive_buffer.sync(0..len as usize).unwrap();
let callbacks = self.callbacks.lock_irq_disabled();
for callback in callbacks.iter() {
let reader = self.receive_buffer.reader().unwrap().limit(len as usize);
callback(reader);
}
receive_queue
.add_dma_buf(&[], &[&self.receive_buffer])
.unwrap();
if receive_queue.should_notify() {
receive_queue.notify();
}
}
}
fn config_space_change(_: &TrapFrame) {

View File

@ -6,9 +6,15 @@ use alloc::{
sync::Arc,
vec::Vec,
};
use core::fmt::Debug;
use core::{fmt::Debug, mem};
use aster_frame::{io_mem::IoMem, offset_of, sync::SpinLock, trap::TrapFrame};
use aster_frame::{
io_mem::IoMem,
offset_of,
sync::SpinLock,
trap::TrapFrame,
vm::{Daddr, DmaDirection, DmaStream, HasDaddr, VmAllocOptions, VmIo, VmReader, PAGE_SIZE},
};
use aster_input::{
key::{Key, KeyStatus},
InputEvent,
@ -16,10 +22,11 @@ use aster_input::{
use aster_util::{field_ptr, safe_ptr::SafePtr};
use bitflags::bitflags;
use log::{debug, info};
use pod::Pod;
use super::{InputConfigSelect, VirtioInputConfig, VirtioInputEvent, QUEUE_EVENT, QUEUE_STATUS};
use crate::{device::VirtioDeviceError, queue::VirtQueue, transport::VirtioTransport};
use crate::{
device::VirtioDeviceError, dma_buf::DmaBuf, queue::VirtQueue, transport::VirtioTransport,
};
bitflags! {
/// The properties of input device.
@ -67,25 +74,25 @@ pub struct InputDevice {
config: SafePtr<VirtioInputConfig, IoMem>,
event_queue: SpinLock<VirtQueue>,
status_queue: VirtQueue,
event_buf: SpinLock<Box<[VirtioInputEvent; QUEUE_SIZE as usize]>>,
event_table: EventTable,
#[allow(clippy::type_complexity)]
callbacks: SpinLock<Vec<Arc<dyn Fn(InputEvent) + Send + Sync + 'static>>>,
transport: Box<dyn VirtioTransport>,
transport: SpinLock<Box<dyn VirtioTransport>>,
}
impl InputDevice {
/// Create a new VirtIO-Input driver.
/// msix_vector_left should at least have one element or n elements where n is the virtqueue amount
pub fn init(mut transport: Box<dyn VirtioTransport>) -> Result<(), VirtioDeviceError> {
let mut event_buf = Box::new([VirtioInputEvent::default(); QUEUE_SIZE as usize]);
let mut event_queue = VirtQueue::new(QUEUE_EVENT, QUEUE_SIZE, transport.as_mut())
.expect("create event virtqueue failed");
let status_queue = VirtQueue::new(QUEUE_STATUS, QUEUE_SIZE, transport.as_mut())
.expect("create status virtqueue failed");
for (i, event) in event_buf.as_mut().iter_mut().enumerate() {
// FIEME: replace slice with a more secure data structure to use dma mapping.
let token = event_queue.add_buf(&[], &[event.as_bytes_mut()]);
let event_table = EventTable::new(QUEUE_SIZE as usize);
for i in 0..event_table.num_events() {
let event_buf = event_table.get(i);
let token = event_queue.add_dma_buf(&[], &[&event_buf]);
match token {
Ok(value) => {
assert_eq!(value, i as u16);
@ -96,14 +103,14 @@ impl InputDevice {
}
}
let mut device = Self {
let device = Arc::new(Self {
config: VirtioInputConfig::new(transport.as_mut()),
event_queue: SpinLock::new(event_queue),
status_queue,
event_buf: SpinLock::new(event_buf),
transport,
event_table,
transport: SpinLock::new(transport),
callbacks: SpinLock::new(Vec::new()),
};
});
let mut raw_name: [u8; 128] = [0; 128];
device.query_config_select(InputConfigSelect::IdName, 0, &mut raw_name);
@ -115,51 +122,49 @@ impl InputDevice {
let input_prop = InputProp::from_bits(prop[0]).unwrap();
debug!("input device prop:{:?}", input_prop);
fn handle_input(_: &TrapFrame) {
debug!("Handle Virtio input interrupt");
let device = aster_input::get_device(super::DEVICE_NAME).unwrap();
device.handle_irq().unwrap();
}
let mut transport = device.transport.lock_irq_disabled();
fn config_space_change(_: &TrapFrame) {
debug!("input device config space change");
}
device
.transport
transport
.register_cfg_callback(Box::new(config_space_change))
.unwrap();
device
.transport
let handle_input = {
let device = device.clone();
move |_: &TrapFrame| device.handle_irq()
};
transport
.register_queue_callback(QUEUE_EVENT, Box::new(handle_input), false)
.unwrap();
device.transport.finish_init();
transport.finish_init();
drop(transport);
aster_input::register_device(super::DEVICE_NAME.to_string(), Arc::new(device));
aster_input::register_device(super::DEVICE_NAME.to_string(), device);
Ok(())
}
/// Pop the pending event.
pub fn pop_pending_event(&self) -> Option<VirtioInputEvent> {
let mut lock = self.event_queue.lock();
if let Ok((token, _)) = lock.pop_used() {
if token >= QUEUE_SIZE {
return None;
}
let event = &mut self.event_buf.lock()[token as usize];
// requeue
// FIEME: replace slice with a more secure data structure to use dma mapping.
if let Ok(new_token) = lock.add_buf(&[], &[event.as_bytes_mut()]) {
// This only works because nothing happen between `pop_used` and `add` that affects
// the list of free descriptors in the queue, so `add` reuses the descriptor which
// was just freed by `pop_used`.
assert_eq!(new_token, token);
return Some(*event);
fn pop_pending_events(&self, handle_event: &impl Fn(&EventBuf) -> bool) {
let mut event_queue = self.event_queue.lock_irq_disabled();
// one interrupt may contain several input events, so it should loop
while let Ok((token, _)) = event_queue.pop_used() {
debug_assert!(token < QUEUE_SIZE);
let event_buf = self.event_table.get(token as usize);
let res = handle_event(&event_buf);
let new_token = event_queue.add_dma_buf(&[], &[&event_buf]).unwrap();
// This only works because nothing happen between `pop_used` and `add` that affects
// the list of free descriptors in the queue, so `add` reuses the descriptor which
// was just freed by `pop_used`.
assert_eq!(new_token, token);
if !res {
break;
}
}
None
}
/// Query a specific piece of information by `select` and `subsel`, and write
@ -181,6 +186,42 @@ impl InputDevice {
size
}
fn handle_irq(&self) {
// Returns ture if there may be more events to handle
let handle_event = |event: &EventBuf| -> bool {
let event: VirtioInputEvent = {
let mut reader = event.reader();
reader.read_val()
};
match event.event_type {
0 => return false,
// Keyboard
1 => {}
// TODO: Support mouse device.
_ => return true,
}
let status = match event.value {
1 => KeyStatus::Pressed,
0 => KeyStatus::Released,
_ => return false,
};
let event = InputEvent::KeyBoard(Key::try_from(event.code).unwrap(), status);
info!("Input Event:{:?}", event);
let callbacks = self.callbacks.lock();
for callback in callbacks.iter() {
callback(event);
}
true
};
self.pop_pending_events(&handle_event);
}
/// Negotiate features for the device specified bits 0~23
pub(crate) fn negotiate_features(features: u64) -> u64 {
assert_eq!(features, 0);
@ -188,37 +229,85 @@ impl InputDevice {
}
}
impl aster_input::InputDevice for InputDevice {
fn handle_irq(&self) -> Option<()> {
// one interrupt may contains serval input, so it should loop
loop {
let Some(event) = self.pop_pending_event() else {
return Some(());
};
match event.event_type {
0 => return Some(()),
// Keyboard
1 => {}
// TODO: Support mouse device.
_ => continue,
}
let status = match event.value {
1 => KeyStatus::Pressed,
0 => KeyStatus::Released,
_ => return Some(()),
};
let event = InputEvent::KeyBoard(Key::try_from(event.code).unwrap(), status);
info!("Input Event:{:?}", event);
/// A event table consists of many event buffers,
/// each of which is large enough to contain a `VirtioInputEvent`.
#[derive(Debug)]
struct EventTable {
stream: DmaStream,
num_events: usize,
}
let callbacks = self.callbacks.lock();
for callback in callbacks.iter() {
callback.call((event,));
}
impl EventTable {
fn new(num_events: usize) -> Self {
assert!(num_events * mem::size_of::<VirtioInputEvent>() <= PAGE_SIZE);
let vm_segment = VmAllocOptions::new(1).alloc_contiguous().unwrap();
let default_event = VirtioInputEvent::default();
for idx in 0..num_events {
let offset = idx * EVENT_SIZE;
vm_segment.write_val(offset, &default_event).unwrap();
}
let stream = DmaStream::map(vm_segment, DmaDirection::FromDevice, false).unwrap();
Self { stream, num_events }
}
fn get(&self, idx: usize) -> EventBuf<'_> {
assert!(idx < self.num_events);
let offset = idx * EVENT_SIZE;
EventBuf {
event_table: self,
offset,
size: EVENT_SIZE,
}
}
const fn num_events(&self) -> usize {
self.num_events
}
}
const EVENT_SIZE: usize = core::mem::size_of::<VirtioInputEvent>();
/// A buffer stores exact one `VirtioInputEvent`
struct EventBuf<'a> {
event_table: &'a EventTable,
offset: usize,
size: usize,
}
impl<'a> HasDaddr for EventBuf<'a> {
fn daddr(&self) -> Daddr {
self.event_table.stream.daddr() + self.offset
}
}
impl<'a> DmaBuf for EventBuf<'a> {
fn len(&self) -> usize {
self.size
}
}
impl<'a> EventBuf<'a> {
fn reader(&self) -> VmReader<'a> {
self.event_table
.stream
.sync(self.offset..self.offset + self.size)
.unwrap();
self.event_table
.stream
.reader()
.unwrap()
.skip(self.offset)
.limit(self.size)
}
}
impl aster_input::InputDevice for InputDevice {
fn register_callbacks(&self, function: &'static (dyn Fn(InputEvent) + Send + Sync)) {
self.callbacks.lock().push(Arc::new(function))
self.callbacks.lock_irq_disabled().push(Arc::new(function))
}
}
@ -228,7 +317,7 @@ impl Debug for InputDevice {
.field("config", &self.config)
.field("event_queue", &self.event_queue)
.field("status_queue", &self.status_queue)
.field("event_buf", &self.event_buf)
.field("event_buf", &self.event_table)
.field("transport", &self.transport)
.finish()
}

View File

@ -5,12 +5,10 @@ use core::{fmt::Debug, hint::spin_loop, mem::size_of};
use aster_frame::{offset_of, sync::SpinLock, trap::TrapFrame};
use aster_network::{
buffer::{RxBuffer, TxBuffer},
AnyNetworkDevice, EthernetAddr, NetDeviceIrqHandler, VirtioNetError,
AnyNetworkDevice, EthernetAddr, NetDeviceIrqHandler, RxBuffer, TxBuffer, VirtioNetError,
};
use aster_util::{field_ptr, slot_vec::SlotVec};
use log::debug;
use pod::Pod;
use smoltcp::phy::{DeviceCapabilities, Medium};
use super::{config::VirtioNetConfig, header::VirtioNetHdr};
@ -60,9 +58,9 @@ impl NetworkDevice {
let mut rx_buffers = SlotVec::new();
for i in 0..QUEUE_SIZE {
let mut rx_buffer = RxBuffer::new(RX_BUFFER_LEN, size_of::<VirtioNetHdr>());
let rx_buffer = RxBuffer::new(size_of::<VirtioNetHdr>());
// FIEME: Replace rx_buffer with VM segment-based data structure to use dma mapping.
let token = recv_queue.add_buf(&[], &[rx_buffer.buf_mut()])?;
let token = recv_queue.add_dma_buf(&[], &[&rx_buffer])?;
assert_eq!(i, token);
assert_eq!(rx_buffers.put(rx_buffer) as u16, i);
}
@ -80,7 +78,7 @@ impl NetworkDevice {
transport,
callbacks: Vec::new(),
};
device.transport.finish_init();
/// Interrupt handler if network device config space changes
fn config_space_change(_: &TrapFrame) {
debug!("network device config space change");
@ -99,6 +97,7 @@ impl NetworkDevice {
.transport
.register_queue_callback(QUEUE_RECV, Box::new(handle_network_event), false)
.unwrap();
device.transport.finish_init();
aster_network::register_device(
super::DEVICE_NAME.to_string(),
@ -109,10 +108,10 @@ impl NetworkDevice {
/// Add a rx buffer to recv queue
/// FIEME: Replace rx_buffer with VM segment-based data structure to use dma mapping.
fn add_rx_buffer(&mut self, mut rx_buffer: RxBuffer) -> Result<(), VirtioNetError> {
fn add_rx_buffer(&mut self, rx_buffer: RxBuffer) -> Result<(), VirtioNetError> {
let token = self
.recv_queue
.add_buf(&[], &[rx_buffer.buf_mut()])
.add_dma_buf(&[], &[&rx_buffer])
.map_err(queue_to_network_error)?;
assert!(self.rx_buffers.put_at(token as usize, rx_buffer).is_none());
if self.recv_queue.should_notify() {
@ -133,18 +132,20 @@ impl NetworkDevice {
rx_buffer.set_packet_len(len as usize);
// FIXME: Ideally, we can reuse the returned buffer without creating new buffer.
// But this requires locking device to be compatible with smoltcp interface.
let new_rx_buffer = RxBuffer::new(RX_BUFFER_LEN, size_of::<VirtioNetHdr>());
let new_rx_buffer = RxBuffer::new(size_of::<VirtioNetHdr>());
self.add_rx_buffer(new_rx_buffer)?;
Ok(rx_buffer)
}
/// Send a packet to network. Return until the request completes.
/// FIEME: Replace tx_buffer with VM segment-based data structure to use dma mapping.
fn send(&mut self, tx_buffer: TxBuffer) -> Result<(), VirtioNetError> {
fn send(&mut self, packet: &[u8]) -> Result<(), VirtioNetError> {
let header = VirtioNetHdr::default();
let tx_buffer = TxBuffer::new(&header, packet);
let token = self
.send_queue
.add_buf(&[header.as_bytes(), tx_buffer.buf()], &[])
.add_dma_buf(&[&tx_buffer], &[])
.map_err(queue_to_network_error)?;
if self.send_queue.should_notify() {
@ -198,8 +199,8 @@ impl AnyNetworkDevice for NetworkDevice {
self.receive()
}
fn send(&mut self, tx_buffer: TxBuffer) -> Result<(), VirtioNetError> {
self.send(tx_buffer)
fn send(&mut self, packet: &[u8]) -> Result<(), VirtioNetError> {
self.send(packet)
}
}

View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: MPL-2.0
use aster_frame::vm::{DmaCoherent, DmaStream, DmaStreamSlice, HasDaddr};
use aster_network::{DmaSegment, RxBuffer, TxBuffer};
/// A DMA-capable buffer.
///
@ -29,3 +30,21 @@ impl DmaBuf for DmaCoherent {
self.nbytes()
}
}
impl DmaBuf for DmaSegment {
fn len(&self) -> usize {
self.size()
}
}
impl DmaBuf for TxBuffer {
fn len(&self) -> usize {
self.nbytes()
}
}
impl DmaBuf for RxBuffer {
fn len(&self) -> usize {
self.buf_len()
}
}

View File

@ -82,10 +82,7 @@ impl VirtQueue {
let desc_size = size_of::<Descriptor>() * size as usize;
let (seg1, seg2) = {
let continue_segment = VmAllocOptions::new(2)
.is_contiguous(true)
.alloc_contiguous()
.unwrap();
let continue_segment = VmAllocOptions::new(2).alloc_contiguous().unwrap();
let seg1 = continue_segment.range(0..1);
let seg2 = continue_segment.range(1..2);
(seg1, seg2)
@ -104,36 +101,18 @@ impl VirtQueue {
}
(
SafePtr::new(
DmaCoherent::map(
VmAllocOptions::new(1)
.is_contiguous(true)
.alloc_contiguous()
.unwrap(),
true,
)
.unwrap(),
DmaCoherent::map(VmAllocOptions::new(1).alloc_contiguous().unwrap(), true)
.unwrap(),
0,
),
SafePtr::new(
DmaCoherent::map(
VmAllocOptions::new(1)
.is_contiguous(true)
.alloc_contiguous()
.unwrap(),
true,
)
.unwrap(),
DmaCoherent::map(VmAllocOptions::new(1).alloc_contiguous().unwrap(), true)
.unwrap(),
0,
),
SafePtr::new(
DmaCoherent::map(
VmAllocOptions::new(1)
.is_contiguous(true)
.alloc_contiguous()
.unwrap(),
true,
)
.unwrap(),
DmaCoherent::map(VmAllocOptions::new(1).alloc_contiguous().unwrap(), true)
.unwrap(),
0,
),
)