From 7d24e632165663c36370507e187140855ac801f8 Mon Sep 17 00:00:00 2001 From: jiangjianfeng Date: Thu, 20 Feb 2025 07:00:53 +0000 Subject: [PATCH] Move network polling code to bottom half --- Cargo.lock | 3 + kernel/comps/network/Cargo.toml | 1 + kernel/comps/network/src/buffer.rs | 7 +- kernel/comps/network/src/dma_pool.rs | 23 ++-- kernel/comps/network/src/lib.rs | 108 +++++++++++------- kernel/comps/softirq/src/softirq_id.rs | 6 + kernel/comps/virtio/Cargo.toml | 1 + .../comps/virtio/src/device/network/device.rs | 13 +-- .../comps/virtio/src/device/socket/buffer.rs | 5 +- kernel/libs/aster-bigtcp/Cargo.toml | 1 + kernel/libs/aster-bigtcp/src/iface/common.rs | 13 ++- .../libs/aster-bigtcp/src/iface/phy/ether.rs | 5 +- .../aster-bigtcp/src/socket/bound/tcp_conn.rs | 7 +- .../src/socket/bound/tcp_listen.rs | 5 +- .../libs/aster-bigtcp/src/socket/bound/udp.rs | 5 +- kernel/src/net/iface/init.rs | 4 +- 16 files changed, 122 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0196be5..148ea0cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,6 +80,7 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" name = "aster-bigtcp" version = "0.1.0" dependencies = [ + "aster-softirq", "bitflags 1.3.2", "jhash", "ostd", @@ -167,6 +168,7 @@ name = "aster-network" version = "0.1.0" dependencies = [ "aster-bigtcp", + "aster-softirq", "bitvec", "component", "ostd", @@ -288,6 +290,7 @@ dependencies = [ "aster-input", "aster-network", "aster-rights", + "aster-softirq", "aster-util", "bitflags 1.3.2", "component", diff --git a/kernel/comps/network/Cargo.toml b/kernel/comps/network/Cargo.toml index 4bb6a3f5..bb272356 100644 --- a/kernel/comps/network/Cargo.toml +++ b/kernel/comps/network/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] aster-bigtcp = { path = "../../libs/aster-bigtcp" } +aster-softirq = { path = "../softirq" } bitvec = { version = "1.0.1", default-features = false, features = ["alloc"] } component = { path = "../../libs/comp-sys/component" } ostd = { path = "../../../ostd" } diff --git a/kernel/comps/network/src/buffer.rs b/kernel/comps/network/src/buffer.rs index d154411f..c8d75d4c 100644 --- a/kernel/comps/network/src/buffer.rs +++ b/kernel/comps/network/src/buffer.rs @@ -2,12 +2,13 @@ use alloc::{collections::linked_list::LinkedList, sync::Arc}; +use aster_softirq::BottomHalfDisabled; use ostd::{ mm::{ Daddr, DmaDirection, DmaStream, FrameAllocOptions, HasDaddr, Infallible, VmReader, VmWriter, PAGE_SIZE, }, - sync::{LocalIrqDisabled, SpinLock}, + sync::SpinLock, Pod, }; use spin::Once; @@ -17,14 +18,14 @@ use crate::dma_pool::{DmaPool, DmaSegment}; pub struct TxBuffer { dma_stream: DmaStream, nbytes: usize, - pool: &'static SpinLock, LocalIrqDisabled>, + pool: &'static SpinLock, BottomHalfDisabled>, } impl TxBuffer { pub fn new( header: &H, packet: &[u8], - pool: &'static SpinLock, LocalIrqDisabled>, + pool: &'static SpinLock, BottomHalfDisabled>, ) -> Self { let header = header.as_bytes(); let nbytes = header.len() + packet.len(); diff --git a/kernel/comps/network/src/dma_pool.rs b/kernel/comps/network/src/dma_pool.rs index 695481e6..448a5742 100644 --- a/kernel/comps/network/src/dma_pool.rs +++ b/kernel/comps/network/src/dma_pool.rs @@ -8,6 +8,7 @@ use alloc::{ }; use core::ops::Range; +use aster_softirq::BottomHalfDisabled; use bitvec::{array::BitArray, prelude::Lsb0}; use ostd::{ mm::{ @@ -34,8 +35,8 @@ pub struct DmaPool { direction: DmaDirection, is_cache_coherent: bool, high_watermark: usize, - avail_pages: SpinLock>>, - all_pages: SpinLock>>, + avail_pages: SpinLock>, BottomHalfDisabled>, + all_pages: SpinLock>, BottomHalfDisabled>, } impl DmaPool { @@ -98,7 +99,7 @@ impl DmaPool { pub fn alloc_segment(self: &Arc) -> Result { // Lock order: pool.avail_pages -> pool.all_pages // pool.avail_pages -> page.allocated_segments - let mut avail_pages = self.avail_pages.disable_irq().lock(); + let mut avail_pages = self.avail_pages.lock(); if avail_pages.is_empty() { /// Allocate a new page let new_page = { @@ -110,7 +111,7 @@ impl DmaPool { pool, )?) }; - let mut all_pages = self.all_pages.disable_irq().lock(); + let mut all_pages = self.all_pages.lock(); avail_pages.push_back(new_page.clone()); all_pages.push_back(new_page); } @@ -125,7 +126,7 @@ impl DmaPool { /// Returns the number of pages in pool fn num_pages(&self) -> usize { - self.all_pages.disable_irq().lock().len() + self.all_pages.lock().len() } /// Return segment size in pool @@ -140,7 +141,7 @@ struct DmaPage { segment_size: usize, // `BitArray` is 64 bits, since each `DmaSegment` is bigger than 64 bytes, // there's no more than `PAGE_SIZE` / 64 = 64 `DmaSegment`s in a `DmaPage`. - allocated_segments: SpinLock, + allocated_segments: SpinLock, pool: Weak, } @@ -167,7 +168,7 @@ impl DmaPage { } fn alloc_segment(self: &Arc) -> Option { - let mut segments = self.allocated_segments.disable_irq().lock(); + let mut segments = self.allocated_segments.lock(); let free_segment_index = get_next_free_index(&segments, self.nr_blocks_per_page())?; segments.set(free_segment_index, true); @@ -190,7 +191,7 @@ impl DmaPage { } fn is_full(&self) -> bool { - let segments = self.allocated_segments.disable_irq().lock(); + let segments = self.allocated_segments.lock(); get_next_free_index(&segments, self.nr_blocks_per_page()).is_none() } } @@ -262,10 +263,10 @@ impl Drop for DmaSegment { // Keep the same lock order as `pool.alloc_segment` // Lock order: pool.avail_pages -> pool.all_pages -> page.allocated_segments - let mut avail_pages = pool.avail_pages.disable_irq().lock(); - let mut all_pages = pool.all_pages.disable_irq().lock(); + let mut avail_pages = pool.avail_pages.lock(); + let mut all_pages = pool.all_pages.lock(); - let mut allocated_segments = page.allocated_segments.disable_irq().lock(); + let mut allocated_segments = page.allocated_segments.lock(); let nr_blocks_per_page = PAGE_SIZE / self.size; let became_avail = get_next_free_index(&allocated_segments, nr_blocks_per_page).is_none(); diff --git a/kernel/comps/network/src/lib.rs b/kernel/comps/network/src/lib.rs index 7ee370ad..3fa1ccd9 100644 --- a/kernel/comps/network/src/lib.rs +++ b/kernel/comps/network/src/lib.rs @@ -16,13 +16,14 @@ use alloc::{collections::BTreeMap, string::String, sync::Arc, vec::Vec}; use core::{any::Any, fmt::Debug}; use aster_bigtcp::device::DeviceCapabilities; +use aster_softirq::{ + softirq_id::{NETWORK_RX_SOFTIRQ_ID, NETWORK_TX_SOFTIRQ_ID}, + BottomHalfDisabled, SoftIrqLine, +}; pub use buffer::{RxBuffer, TxBuffer, RX_BUFFER_POOL, TX_BUFFER_LEN}; use component::{init_component, ComponentInitError}; pub use dma_pool::DmaSegment; -use ostd::{ - sync::{LocalIrqDisabled, SpinLock}, - Pod, -}; +use ostd::{sync::SpinLock, Pod}; use spin::Once; #[derive(Debug, Clone, Copy, Pod)] @@ -66,11 +67,11 @@ pub trait AnyNetworkDevice: Send + Sync + Any + Debug { fn notify_poll_end(&mut self); } -pub trait NetDeviceIrqHandler = Fn() + Send + Sync + 'static; +pub trait NetDeviceCallback = Fn() + Send + Sync + 'static; pub fn register_device( name: String, - device: Arc>, + device: Arc>, ) { COMPONENT .get() @@ -80,7 +81,7 @@ pub fn register_device( .insert(name, NetworkDeviceIrqCallbackSet::new(device)); } -pub fn get_device(str: &str) -> Option>> { +pub fn get_device(str: &str) -> Option>> { let table = COMPONENT.get().unwrap().network_device_table.lock(); let callbacks = table.get(str)?; Some(callbacks.device.clone()) @@ -88,9 +89,9 @@ pub fn get_device(str: &str) -> Option Vec<(String, NetworkDeviceRef)> { let network_devs = COMPONENT.get().unwrap().network_device_table.lock(); network_devs @@ -148,33 +170,31 @@ pub fn all_devices() -> Vec<(String, NetworkDeviceRef)> { } static COMPONENT: Once = Once::new(); -pub(crate) static NETWORK_IRQ_HANDLERS: Once< - SpinLock>, LocalIrqDisabled>, -> = Once::new(); #[init_component] fn init() -> Result<(), ComponentInitError> { let a = Component::init()?; COMPONENT.call_once(|| a); - NETWORK_IRQ_HANDLERS.call_once(|| SpinLock::new(Vec::new())); + SoftIrqLine::get(NETWORK_TX_SOFTIRQ_ID).enable(handle_tx_softirq); + SoftIrqLine::get(NETWORK_RX_SOFTIRQ_ID).enable(handle_rx_softirq); buffer::init(); Ok(()) } -type NetDeviceIrqHandlerListRef = - Arc>, LocalIrqDisabled>>; -type NetworkDeviceRef = Arc>; +type NetDeviceCallbackListRef = Arc>, BottomHalfDisabled>>; +type NetworkDeviceRef = Arc>; struct Component { /// Device list, the key is device name, value is (callbacks, device); - network_device_table: SpinLock, LocalIrqDisabled>, + network_device_table: + SpinLock, BottomHalfDisabled>, } /// The send callbacks and recv callbacks for a network device struct NetworkDeviceIrqCallbackSet { device: NetworkDeviceRef, - recv_callbacks: NetDeviceIrqHandlerListRef, - send_callbacks: NetDeviceIrqHandlerListRef, + recv_callbacks: NetDeviceCallbackListRef, + send_callbacks: NetDeviceCallbackListRef, } impl NetworkDeviceIrqCallbackSet { diff --git a/kernel/comps/softirq/src/softirq_id.rs b/kernel/comps/softirq/src/softirq_id.rs index db0f8a35..30b5743a 100644 --- a/kernel/comps/softirq/src/softirq_id.rs +++ b/kernel/comps/softirq/src/softirq_id.rs @@ -11,3 +11,9 @@ pub const TIMER_SOFTIRQ_ID: u8 = 1; /// The corresponding softirq line is used to schedule general taskless jobs. pub const TASKLESS_SOFTIRQ_ID: u8 = 2; + +/// The corresponding softirq line is used to handle transmission network events. +pub const NETWORK_TX_SOFTIRQ_ID: u8 = 3; + +/// The corresponding softirq line is used to handle reception network events. +pub const NETWORK_RX_SOFTIRQ_ID: u8 = 4; diff --git a/kernel/comps/virtio/Cargo.toml b/kernel/comps/virtio/Cargo.toml index 4725f1ee..8c546313 100644 --- a/kernel/comps/virtio/Cargo.toml +++ b/kernel/comps/virtio/Cargo.toml @@ -15,6 +15,7 @@ aster-console = { path = "../console" } aster-util = { path = "../../libs/aster-util" } aster-rights = { path = "../../libs/aster-rights" } aster-bigtcp = { path = "../../libs/aster-bigtcp" } +aster-softirq = { path = "../softirq"} id-alloc = { path = "../../../ostd/libs/id-alloc" } typeflags-util = { path = "../../libs/typeflags-util" } ostd = { path = "../../../ostd" } diff --git a/kernel/comps/virtio/src/device/network/device.rs b/kernel/comps/virtio/src/device/network/device.rs index f734a5fa..2a277444 100644 --- a/kernel/comps/virtio/src/device/network/device.rs +++ b/kernel/comps/virtio/src/device/network/device.rs @@ -9,13 +9,10 @@ use aster_bigtcp::device::{Checksum, DeviceCapabilities, Medium}; use aster_network::{ AnyNetworkDevice, EthernetAddr, RxBuffer, TxBuffer, VirtioNetError, RX_BUFFER_POOL, }; +use aster_softirq::BottomHalfDisabled; use aster_util::slot_vec::SlotVec; use log::{debug, warn}; -use ostd::{ - mm::DmaStream, - sync::{LocalIrqDisabled, SpinLock}, - trap::TrapFrame, -}; +use ostd::{mm::DmaStream, sync::SpinLock, trap::TrapFrame}; use super::{config::VirtioNetConfig, header::VirtioNetHdr}; use crate::{ @@ -127,10 +124,10 @@ impl NetworkDevice { /// Interrupt handlers if network device receives/sends some packet fn handle_send_event(_: &TrapFrame) { - aster_network::handle_send_irq(super::DEVICE_NAME); + aster_network::raise_send_softirq(); } fn handle_recv_event(_: &TrapFrame) { - aster_network::handle_recv_irq(super::DEVICE_NAME); + aster_network::raise_receive_softirq(); } device @@ -363,7 +360,7 @@ impl Debug for NetworkDevice { } } -static TX_BUFFER_POOL: SpinLock, LocalIrqDisabled> = +static TX_BUFFER_POOL: SpinLock, BottomHalfDisabled> = SpinLock::new(LinkedList::new()); const QUEUE_RECV: u16 = 0; diff --git a/kernel/comps/virtio/src/device/socket/buffer.rs b/kernel/comps/virtio/src/device/socket/buffer.rs index 7b2faa40..863a4e13 100644 --- a/kernel/comps/virtio/src/device/socket/buffer.rs +++ b/kernel/comps/virtio/src/device/socket/buffer.rs @@ -3,15 +3,16 @@ use alloc::{collections::linked_list::LinkedList, sync::Arc}; use aster_network::dma_pool::DmaPool; +use aster_softirq::BottomHalfDisabled; use ostd::{ mm::{DmaDirection, DmaStream}, - sync::{LocalIrqDisabled, SpinLock}, + sync::SpinLock, }; use spin::Once; const RX_BUFFER_LEN: usize = 4096; pub static RX_BUFFER_POOL: Once> = Once::new(); -pub static TX_BUFFER_POOL: Once, LocalIrqDisabled>> = Once::new(); +pub static TX_BUFFER_POOL: Once, BottomHalfDisabled>> = Once::new(); pub fn init() { const POOL_INIT_SIZE: usize = 32; diff --git a/kernel/libs/aster-bigtcp/Cargo.toml b/kernel/libs/aster-bigtcp/Cargo.toml index f188fd90..07d10ba1 100644 --- a/kernel/libs/aster-bigtcp/Cargo.toml +++ b/kernel/libs/aster-bigtcp/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +aster-softirq = { path = "../../comps/softirq" } bitflags = "1.3" jhash = { path = "../jhash" } ostd = { path = "../../../ostd" } diff --git a/kernel/libs/aster-bigtcp/src/iface/common.rs b/kernel/libs/aster-bigtcp/src/iface/common.rs index b29883e9..48456020 100644 --- a/kernel/libs/aster-bigtcp/src/iface/common.rs +++ b/kernel/libs/aster-bigtcp/src/iface/common.rs @@ -7,7 +7,8 @@ use alloc::{ vec::Vec, }; -use ostd::sync::{LocalIrqDisabled, SpinLock, SpinLockGuard}; +use aster_softirq::BottomHalfDisabled; +use ostd::sync::{SpinLock, SpinLockGuard}; use smoltcp::{ iface::{packet::Packet, Context}, phy::Device, @@ -30,9 +31,9 @@ use crate::{ pub struct IfaceCommon { name: String, - interface: SpinLock, LocalIrqDisabled>, - used_ports: SpinLock, LocalIrqDisabled>, - sockets: SpinLock, LocalIrqDisabled>, + interface: SpinLock, BottomHalfDisabled>, + used_ports: SpinLock, BottomHalfDisabled>, + sockets: SpinLock, BottomHalfDisabled>, sched_poll: E::ScheduleNextPoll, } @@ -67,12 +68,12 @@ impl IfaceCommon { // Lock order: `interface` -> `sockets` impl IfaceCommon { /// Acquires the lock to the interface. - pub(crate) fn interface(&self) -> SpinLockGuard<'_, PollableIface, LocalIrqDisabled> { + pub(crate) fn interface(&self) -> SpinLockGuard<'_, PollableIface, BottomHalfDisabled> { self.interface.lock() } /// Acquires the lock to the socket table. - pub(crate) fn sockets(&self) -> SpinLockGuard<'_, SocketTable, LocalIrqDisabled> { + pub(crate) fn sockets(&self) -> SpinLockGuard<'_, SocketTable, BottomHalfDisabled> { self.sockets.lock() } } diff --git a/kernel/libs/aster-bigtcp/src/iface/phy/ether.rs b/kernel/libs/aster-bigtcp/src/iface/phy/ether.rs index c4515527..949bd2c2 100644 --- a/kernel/libs/aster-bigtcp/src/iface/phy/ether.rs +++ b/kernel/libs/aster-bigtcp/src/iface/phy/ether.rs @@ -2,7 +2,8 @@ use alloc::{collections::btree_map::BTreeMap, string::String, sync::Arc}; -use ostd::sync::{LocalIrqDisabled, SpinLock}; +use aster_softirq::BottomHalfDisabled; +use ostd::sync::SpinLock; use smoltcp::{ iface::{packet::Packet, Config, Context}, phy::{DeviceCapabilities, TxToken}, @@ -25,7 +26,7 @@ pub struct EtherIface { driver: D, common: IfaceCommon, ether_addr: EthernetAddress, - arp_table: SpinLock, LocalIrqDisabled>, + arp_table: SpinLock, BottomHalfDisabled>, } impl EtherIface { diff --git a/kernel/libs/aster-bigtcp/src/socket/bound/tcp_conn.rs b/kernel/libs/aster-bigtcp/src/socket/bound/tcp_conn.rs index ec60ff28..26cc3e22 100644 --- a/kernel/libs/aster-bigtcp/src/socket/bound/tcp_conn.rs +++ b/kernel/libs/aster-bigtcp/src/socket/bound/tcp_conn.rs @@ -6,7 +6,8 @@ use alloc::{ }; use core::ops::{Deref, DerefMut}; -use ostd::sync::{LocalIrqDisabled, SpinLock, SpinLockGuard}; +use aster_softirq::BottomHalfDisabled; +use ostd::sync::{SpinLock, SpinLockGuard}; use smoltcp::{ socket::{tcp::State, PollAt}, time::Duration, @@ -34,7 +35,7 @@ pub type TcpConnection = Socket, E>; /// States needed by [`TcpConnectionBg`]. pub struct TcpConnectionInner { - socket: SpinLock, LocalIrqDisabled>, + socket: SpinLock, BottomHalfDisabled>, poll_key: PollKey, connection_key: ConnectionKey, } @@ -243,7 +244,7 @@ impl TcpConnectionInner { } } - pub(super) fn lock(&self) -> SpinLockGuard, LocalIrqDisabled> { + pub(super) fn lock(&self) -> SpinLockGuard, BottomHalfDisabled> { self.socket.lock() } } diff --git a/kernel/libs/aster-bigtcp/src/socket/bound/tcp_listen.rs b/kernel/libs/aster-bigtcp/src/socket/bound/tcp_listen.rs index 0d3a5186..25617256 100644 --- a/kernel/libs/aster-bigtcp/src/socket/bound/tcp_listen.rs +++ b/kernel/libs/aster-bigtcp/src/socket/bound/tcp_listen.rs @@ -2,7 +2,8 @@ use alloc::{boxed::Box, collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; -use ostd::sync::{LocalIrqDisabled, SpinLock}; +use aster_softirq::BottomHalfDisabled; +use ostd::sync::SpinLock; use smoltcp::{ socket::PollAt, time::Duration, @@ -35,7 +36,7 @@ pub struct TcpBacklog { /// States needed by [`TcpListenerBg`]. pub struct TcpListenerInner { - pub(super) backlog: SpinLock, LocalIrqDisabled>, + pub(super) backlog: SpinLock, BottomHalfDisabled>, listener_key: ListenerKey, } diff --git a/kernel/libs/aster-bigtcp/src/socket/bound/udp.rs b/kernel/libs/aster-bigtcp/src/socket/bound/udp.rs index e3c5ccba..ebebf58a 100644 --- a/kernel/libs/aster-bigtcp/src/socket/bound/udp.rs +++ b/kernel/libs/aster-bigtcp/src/socket/bound/udp.rs @@ -3,7 +3,8 @@ use alloc::{boxed::Box, sync::Arc}; use core::sync::atomic::{AtomicBool, Ordering}; -use ostd::sync::{LocalIrqDisabled, SpinLock}; +use aster_softirq::BottomHalfDisabled; +use ostd::sync::SpinLock; use smoltcp::{ iface::Context, socket::udp::UdpMetadata, @@ -22,7 +23,7 @@ pub type UdpSocket = Socket; /// States needed by [`UdpSocketBg`]. pub struct UdpSocketInner { - socket: SpinLock, LocalIrqDisabled>, + socket: SpinLock, BottomHalfDisabled>, need_dispatch: AtomicBool, } diff --git a/kernel/src/net/iface/init.rs b/kernel/src/net/iface/init.rs index e41694b9..03f14f9c 100644 --- a/kernel/src/net/iface/init.rs +++ b/kernel/src/net/iface/init.rs @@ -3,7 +3,7 @@ use alloc::{borrow::ToOwned, sync::Arc}; use aster_bigtcp::device::WithDevice; -use ostd::sync::LocalIrqDisabled; +use aster_softirq::BottomHalfDisabled; use spin::Once; use super::{poll::poll_ifaces, Iface}; @@ -52,7 +52,7 @@ fn new_virtio() -> Option> { let ether_addr = virtio_net.lock().mac_addr().0; - struct Wrapper(Arc>); + struct Wrapper(Arc>); impl WithDevice for Wrapper { type Device = dyn AnyNetworkDevice;