diff --git a/kernel/aster-nix/src/taskless.rs b/kernel/aster-nix/src/taskless.rs index d25801da5..14bd44fa9 100644 --- a/kernel/aster-nix/src/taskless.rs +++ b/kernel/aster-nix/src/taskless.rs @@ -5,11 +5,12 @@ use alloc::{boxed::Box, sync::Arc}; use core::{ cell::RefCell, + ops::DerefMut, sync::atomic::{AtomicBool, Ordering}, }; use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListAtomicLink}; -use ostd::{cpu_local, sync::SpinLock, trap::SoftIrqLine, CpuLocal}; +use ostd::{cpu_local, trap::SoftIrqLine, CpuLocal}; use crate::softirq_id::{TASKLESS_SOFTIRQ_ID, TASKLESS_URGENT_SOFTIRQ_ID}; @@ -65,8 +66,8 @@ pub struct Taskless { intrusive_adapter!(TasklessAdapter = Arc: Taskless { link: LinkedListAtomicLink }); cpu_local! { - static TASKLESS_LIST: SpinLock> = SpinLock::new(LinkedList::new(TasklessAdapter::NEW)); - static TASKLESS_URGENT_LIST: SpinLock> = SpinLock::new(LinkedList::new(TasklessAdapter::NEW)); + static TASKLESS_LIST: RefCell> = RefCell::new(LinkedList::new(TasklessAdapter::NEW)); + static TASKLESS_URGENT_LIST: RefCell> = RefCell::new(LinkedList::new(TasklessAdapter::NEW)); } impl Taskless { @@ -121,7 +122,7 @@ impl Taskless { fn do_schedule( taskless: &Arc, - taskless_list: &'static CpuLocal>>, + taskless_list: &'static CpuLocal>>, ) { if taskless.is_disabled.load(Ordering::Acquire) { return; @@ -133,10 +134,10 @@ fn do_schedule( { return; } - - CpuLocal::borrow_with(taskless_list, |list| { - list.lock_irq_disabled().push_front(taskless.clone()); - }); + taskless_list + .borrow_irq_disabled() + .borrow_mut() + .push_front(taskless.clone()); } pub(super) fn init() { @@ -155,13 +156,14 @@ pub(super) fn init() { /// If the `Taskless` is ready to be executed, it will be set to not scheduled /// and can be scheduled again. fn taskless_softirq_handler( - taskless_list: &'static CpuLocal>>, + taskless_list: &'static CpuLocal>>, softirq_id: u8, ) { - let mut processing_list = CpuLocal::borrow_with(taskless_list, |list| { - let mut list_mut = list.lock_irq_disabled(); - LinkedList::take(&mut list_mut) - }); + let mut processing_list = { + let guard = taskless_list.borrow_irq_disabled(); + let mut list_mut = guard.borrow_mut(); + LinkedList::take(list_mut.deref_mut()) + }; while let Some(taskless) = processing_list.pop_back() { if taskless @@ -169,10 +171,11 @@ fn taskless_softirq_handler( .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) .is_err() { - CpuLocal::borrow_with(taskless_list, |list| { - list.lock_irq_disabled().push_front(taskless); - SoftIrqLine::get(softirq_id).raise(); - }); + taskless_list + .borrow_irq_disabled() + .borrow_mut() + .push_front(taskless); + SoftIrqLine::get(softirq_id).raise(); continue; } diff --git a/kernel/aster-nix/src/time/clocks/system_wide.rs b/kernel/aster-nix/src/time/clocks/system_wide.rs index dbd2a42cf..3be868ee9 100644 --- a/kernel/aster-nix/src/time/clocks/system_wide.rs +++ b/kernel/aster-nix/src/time/clocks/system_wide.rs @@ -4,7 +4,7 @@ use alloc::sync::Arc; use core::time::Duration; use aster_time::read_monotonic_time; -use ostd::{arch::timer::Jiffies, cpu_local, sync::SpinLock, CpuLocal}; +use ostd::{arch::timer::Jiffies, cpu_local, sync::SpinLock}; use paste::paste; use spin::Once; @@ -215,9 +215,7 @@ macro_rules! define_timer_managers { let clock = paste! {[<$clock_id _INSTANCE>].get().unwrap().clone()}; let clock_manager = TimerManager::new(clock); paste! { - CpuLocal::borrow_with(&[<$clock_id _MANAGER>], |manager| { - manager.call_once(|| clock_manager.clone()); - }); + [<$clock_id _MANAGER>].call_once(|| clock_manager.clone()); } let callback = move || { clock_manager.process_expired_timers(); diff --git a/osdk/src/base_crate/x86_64.ld.template b/osdk/src/base_crate/x86_64.ld.template index 053d965e0..18539eb1f 100644 --- a/osdk/src/base_crate/x86_64.ld.template +++ b/osdk/src/base_crate/x86_64.ld.template @@ -52,6 +52,16 @@ SECTIONS . = DATA_SEGMENT_RELRO_END(0, .); .data : AT(ADDR(.data) - KERNEL_VMA) { *(.data .data.*) } + + # The CPU local data storage. It is readable and writable for the bootstrap + # processor, while it would be copied to other dynamically allocated memory + # areas for the application processors. + .cpu_local : AT(ADDR(.cpu_local) - KERNEL_VMA) { + __cpu_local_start = .; + KEEP(*(SORT(.cpu_local))) + __cpu_local_end = .; + } + .bss : AT(ADDR(.bss) - KERNEL_VMA) { __bss = .; *(.bss .bss.*) *(COMMON) diff --git a/ostd/src/arch/x86/cpu.rs b/ostd/src/arch/x86/cpu.rs index 5d1772476..da7beea74 100644 --- a/ostd/src/arch/x86/cpu.rs +++ b/ostd/src/arch/x86/cpu.rs @@ -17,7 +17,10 @@ use log::debug; #[cfg(feature = "intel_tdx")] use tdx_guest::tdcall; use trapframe::{GeneralRegs, UserContext as RawUserContext}; -use x86_64::registers::rflags::RFlags; +use x86_64::registers::{ + rflags::RFlags, + segmentation::{Segment64, FS}, +}; #[cfg(feature = "intel_tdx")] use crate::arch::tdx_guest::{handle_virtual_exception, TdxTrapFrame}; @@ -669,3 +672,22 @@ impl Default for FpRegs { struct FxsaveArea { data: [u8; 512], // 512 bytes } + +/// Sets the base address for the CPU local storage by writing to the FS base model-specific register. +/// This operation is marked as `unsafe` because it directly interfaces with low-level CPU registers. +/// +/// # Safety +/// +/// - This function is safe to call provided that the FS register is dedicated entirely for CPU local storage +/// and is not concurrently accessed for other purposes. +/// - The caller must ensure that `addr` is a valid address and properly aligned, as required by the CPU. +/// - This function should only be called in contexts where the CPU is in a state to accept such changes, +/// such as during processor initialization. +pub(crate) unsafe fn set_cpu_local_base(addr: u64) { + FS::write_base(x86_64::addr::VirtAddr::new(addr)); +} + +/// Gets the base address for the CPU local storage by reading the FS base model-specific register. +pub(crate) fn get_cpu_local_base() -> u64 { + FS::read_base().as_u64() +} diff --git a/ostd/src/arch/x86/timer/mod.rs b/ostd/src/arch/x86/timer/mod.rs index 3eb48d5c9..b7dd95ea8 100644 --- a/ostd/src/arch/x86/timer/mod.rs +++ b/ostd/src/arch/x86/timer/mod.rs @@ -15,7 +15,7 @@ use spin::Once; use trapframe::TrapFrame; use self::apic::APIC_TIMER_CALLBACK; -use crate::{arch::x86::kernel, cpu_local, trap::IrqLine, CpuLocal}; +use crate::{arch::x86::kernel, cpu_local, trap::IrqLine}; /// The timer frequency (Hz). Here we choose 1000Hz since 1000Hz is easier for unit conversion and /// convenient for timer. What's more, the frequency cannot be set too high or too low, 1000Hz is @@ -57,19 +57,20 @@ pub fn register_callback(func: F) where F: Fn() + Sync + Send + 'static, { - CpuLocal::borrow_with(&INTERRUPT_CALLBACKS, |callbacks| { - callbacks.borrow_mut().push(Box::new(func)); - }); + INTERRUPT_CALLBACKS + .borrow_irq_disabled() + .borrow_mut() + .push(Box::new(func)); } fn timer_callback(_: &TrapFrame) { jiffies::ELAPSED.fetch_add(1, Ordering::SeqCst); - CpuLocal::borrow_with(&INTERRUPT_CALLBACKS, |callbacks| { - for callback in callbacks.borrow().iter() { - (callback)(); - } - }); + let callbacks_guard = INTERRUPT_CALLBACKS.borrow_irq_disabled(); + for callback in callbacks_guard.borrow().iter() { + (callback)(); + } + drop(callbacks_guard); if APIC_TIMER_CALLBACK.is_completed() { APIC_TIMER_CALLBACK.get().unwrap().call(()); diff --git a/ostd/src/cpu.rs b/ostd/src/cpu.rs deleted file mode 100644 index 0e9ad6044..000000000 --- a/ostd/src/cpu.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 - -//! CPU. - -use core::{cell::UnsafeCell, ops::Deref}; - -use crate::trap::disable_local; - -cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")]{ - pub use trapframe::GeneralRegs; - pub use crate::arch::x86::cpu::*; - } -} - -/// Defines a CPU-local variable. -/// -/// # Example -/// -/// ```rust -/// use crate::cpu_local; -/// use core::cell::RefCell; -/// -/// cpu_local! { -/// static FOO: RefCell = RefCell::new(1); -/// -/// #[allow(unused)] -/// pub static BAR: RefCell = RefCell::new(1.0); -/// } -/// CpuLocal::borrow_with(&FOO, |val| { -/// println!("FOO VAL: {:?}", *val); -/// }) -/// -/// ``` -#[macro_export] -macro_rules! cpu_local { - // empty - () => {}; - - // multiple declarations - ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => { - #[allow(clippy::macro_metavars_in_unsafe)] - $(#[$attr])* $vis static $name: $crate::CpuLocal<$t> = unsafe { $crate::CpuLocal::new($init) }; - $crate::cpu_local!($($rest)*); - }; - - // single declaration - ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr) => ( - // TODO: reimplement cpu-local variable to support multi-core - $(#[$attr])* $vis static $name: $crate::CpuLocal<$t> = $crate::CpuLocal::new($init); - ); -} - -/// CPU-local objects. -/// -/// A CPU-local object only gives you immutable references to the underlying value. -/// To mutate the value, one can use atomic values (e.g., [`AtomicU32`]) or internally mutable -/// objects (e.g., [`RefCell`]). -/// -/// The `CpuLocal` can be used directly. -/// Otherwise, the `CpuLocal` must be used through [`borrow_with`]. -/// -/// TODO: re-implement `CpuLocal` -/// -/// [`AtomicU32`]: core::sync::atomic::AtomicU32 -/// [`RefCell`]: core::cell::RefCell -/// [`borrow_with`]: CpuLocal::borrow_with -pub struct CpuLocal(UnsafeCell); - -// SAFETY: At any given time, only one task can access the inner value T of a cpu-local variable. -unsafe impl Sync for CpuLocal {} - -impl CpuLocal { - /// Initialize CPU-local object - /// Developer cannot construct a valid CpuLocal object arbitrarily - #[allow(clippy::missing_safety_doc)] - pub const unsafe fn new(val: T) -> Self { - Self(UnsafeCell::new(val)) - } - - /// Borrow an immutable reference to the underlying value and feed it to a closure. - /// - /// During the execution of the closure, local IRQs are disabled. This ensures that - /// the CPU-local object is only accessed by the current task or IRQ handler. - /// As local IRQs are disabled, one should keep the closure as short as possible. - pub fn borrow_with<'a, U, F: FnOnce(&'a T) -> U>(this: &'a Self, f: F) -> U { - // FIXME: implement disable preemption - // Disable interrupts when accessing cpu-local variable - let _guard = disable_local(); - // SAFETY: Now that the local IRQs are disabled, this CPU-local object can only be - // accessed by the current task/thread. So it is safe to get its immutable reference - // regardless of whether `T` implements `Sync` or not. - let val_ref = unsafe { this.do_borrow() }; - f(val_ref) - } - - unsafe fn do_borrow(&self) -> &T { - &*self.0.get() - } -} - -impl Deref for CpuLocal { - type Target = T; - - fn deref(&self) -> &T { - unsafe { &*self.0.get() } - } -} diff --git a/ostd/src/cpu/cpu_local.rs b/ostd/src/cpu/cpu_local.rs new file mode 100644 index 000000000..ef3f96c50 --- /dev/null +++ b/ostd/src/cpu/cpu_local.rs @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! CPU local storage. +//! +//! This module provides a mechanism to define CPU-local objects. +//! +//! This is acheived by placing the CPU-local objects in a special section +//! `.cpu_local`. The bootstrap processor (BSP) uses the objects linked in this +//! section, and these objects are copied to dynamically allocated local +//! storage of each application processors (AP) during the initialization +//! process. +//! +//! Such a mechanism exploits the fact that constant values of non-[`Copy`] +//! types can be bitwise copied. For example, a [`Option`] object, though +//! being not [`Copy`], have a constant constructor [`Option::None`] that +//! produces a value that can be bitwise copied to create a new instance. +//! [`alloc::sync::Arc`] however, don't have such a constructor, and thus cannot +//! be directly used as a CPU-local object. Wrapping it in a type that has a +//! constant constructor, like [`Option`], can make it CPU-local. + +use core::ops::Deref; + +use crate::{ + cpu::{get_cpu_local_base, set_cpu_local_base}, + trap::{disable_local, DisabledLocalIrqGuard}, +}; + +/// Defines a CPU-local variable. +/// +/// # Example +/// +/// ```rust +/// use crate::cpu_local; +/// use core::cell::RefCell; +/// +/// cpu_local! { +/// static FOO: RefCell = RefCell::new(1); +/// +/// #[allow(unused)] +/// pub static BAR: RefCell = RefCell::new(1.0); +/// } +/// +/// println!("FOO VAL: {:?}", *FOO.borrow()); +/// ``` +#[macro_export] +macro_rules! cpu_local { + ($( $(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; )*) => { + $( + #[link_section = ".cpu_local"] + $(#[$attr])* $vis static $name: $crate::CpuLocal<$t> = { + let val = $init; + // SAFETY: The CPU local variable instantiated is statically + // stored in the special `.cpu_local` section. + unsafe { + $crate::CpuLocal::__new(val) + } + }; + )* + }; +} + +/// CPU-local objects. +/// +/// A CPU-local object only gives you immutable references to the underlying value. +/// To mutate the value, one can use atomic values (e.g., [`AtomicU32`]) or internally mutable +/// objects (e.g., [`RefCell`]). +/// +/// [`AtomicU32`]: core::sync::atomic::AtomicU32 +/// [`RefCell`]: core::cell::RefCell +pub struct CpuLocal(T); + +// SAFETY: At any given time, only one task can access the inner value T +// of a cpu-local variable even if `T` is not `Sync`. +unsafe impl Sync for CpuLocal {} + +// Prevent valid instances of CpuLocal from being copied to any memory +// area outside the .cpu_local section. +impl !Copy for CpuLocal {} +impl !Clone for CpuLocal {} + +// In general, it does not make any sense to send instances of CpuLocal to +// other tasks as they should live on other CPUs to make sending useful. +impl !Send for CpuLocal {} + +impl CpuLocal { + /// Initialize a CPU-local object. + /// + /// Please do not call this function directly. Instead, use the + /// `cpu_local!` macro. + /// + /// # Safety + /// + /// The caller should ensure that the object initialized by this + /// function resides in the `.cpu_local` section. Otherwise the + /// behavior is undefined. + #[doc(hidden)] + pub const unsafe fn __new(val: T) -> Self { + Self(val) + } + + /// Get access to the underlying value with IRQs disabled. + /// + /// By this method, you can borrow a reference to the underlying value + /// even if `T` is not `Sync`. Because that it is per-CPU and IRQs are + /// disabled, no other running task can access it. + pub fn borrow_irq_disabled(&self) -> CpuLocalDerefGuard<'_, T> { + CpuLocalDerefGuard { + cpu_local: self, + _guard: disable_local(), + } + } + + /// Get access to the underlying value through a raw pointer. + /// + /// This function calculates the virtual address of the CPU-local object based on the per- + /// cpu base address and the offset in the BSP. + fn get(&self) -> *const T { + let offset = { + let bsp_va = self as *const _ as usize; + let bsp_base = __cpu_local_start as usize; + // The implementation should ensure that the CPU-local object resides in the `.cpu_local`. + debug_assert!(bsp_va + core::mem::size_of::() <= __cpu_local_end as usize); + + bsp_va - bsp_base as usize + }; + + let local_base = get_cpu_local_base() as usize; + let local_va = local_base + offset; + + // A sanity check about the alignment. + debug_assert_eq!(local_va % core::mem::align_of::(), 0); + + local_va as *mut T + } +} + +// Considering a preemptive kernel, a CPU-local object may be dereferenced +// when another task tries to access it. So, we need to ensure that `T` is +// `Sync` before allowing it to be dereferenced. +impl Deref for CpuLocal { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: it should be properly initialized before accesses. + // And we do not create a mutable reference over it. It is + // `Sync` so it can be referenced from this task. + unsafe { &*self.get() } + } +} + +/// A guard for accessing the CPU-local object. +/// +/// It ensures that the CPU-local object is accessed with IRQs +/// disabled. It is created by [`CpuLocal::borrow_irq_disabled`]. +/// Do not hold this guard for a long time. +#[must_use] +pub struct CpuLocalDerefGuard<'a, T> { + cpu_local: &'a CpuLocal, + _guard: DisabledLocalIrqGuard, +} + +impl Deref for CpuLocalDerefGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: it should be properly initialized before accesses. + // And we do not create a mutable reference over it. The IRQs + // are disabled so it can be referenced from this task. + unsafe { &*self.cpu_local.get() } + } +} + +/// Initializes the CPU local data for the bootstrap processor (BSP). +/// +/// # Safety +/// +/// This function can only called on the BSP, for once. +/// +/// It must be guaranteed that the BSP will not access local data before +/// this function being called, otherwise copying non-constant values +/// will result in pretty bad undefined behavior. +pub unsafe fn init_on_bsp() { + let start_base_va = __cpu_local_start as usize as u64; + set_cpu_local_base(start_base_va); +} + +// These symbols are provided by the linker script. +extern "C" { + fn __cpu_local_start(); + fn __cpu_local_end(); +} + +#[cfg(ktest)] +mod test { + use core::{ + cell::RefCell, + sync::atomic::{AtomicU8, Ordering}, + }; + + use ostd_macros::ktest; + + use super::*; + + #[ktest] + fn test_cpu_local() { + cpu_local! { + static FOO: RefCell = RefCell::new(1); + static BAR: AtomicU8 = AtomicU8::new(3); + } + for _ in 0..10 { + let foo_guard = FOO.borrow_irq_disabled(); + assert_eq!(*foo_guard.borrow(), 1); + *foo_guard.borrow_mut() = 2; + drop(foo_guard); + for _ in 0..10 { + assert_eq!(BAR.load(Ordering::Relaxed), 3); + BAR.store(4, Ordering::Relaxed); + assert_eq!(BAR.load(Ordering::Relaxed), 4); + BAR.store(3, Ordering::Relaxed); + } + let foo_guard = FOO.borrow_irq_disabled(); + assert_eq!(*foo_guard.borrow(), 2); + *foo_guard.borrow_mut() = 1; + drop(foo_guard); + } + } +} diff --git a/ostd/src/cpu/mod.rs b/ostd/src/cpu/mod.rs new file mode 100644 index 000000000..63f84e9fa --- /dev/null +++ b/ostd/src/cpu/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! CPU-related definitions. + +pub mod cpu_local; + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")]{ + pub use trapframe::GeneralRegs; + pub use crate::arch::x86::cpu::*; + } +} diff --git a/ostd/src/lib.rs b/ostd/src/lib.rs index ee6935bd5..4e7ab6e51 100644 --- a/ostd/src/lib.rs +++ b/ostd/src/lib.rs @@ -47,7 +47,7 @@ pub use ostd_macros::main; #[cfg(feature = "intel_tdx")] use tdx_guest::init_tdx; -pub use self::{cpu::CpuLocal, error::Error, prelude::Result}; +pub use self::{cpu::cpu_local::CpuLocal, error::Error, prelude::Result}; /// Initializes OSTD. /// @@ -77,6 +77,9 @@ pub fn init() { mm::page::allocator::init(); mm::kspace::init_boot_page_table(); mm::kspace::init_kernel_page_table(mm::init_page_meta()); + // SAFETY: no CPU local objects have been accessed by this far. And + // we are on the BSP. + unsafe { cpu::cpu_local::init_on_bsp() }; mm::misc_init(); trap::init(); diff --git a/ostd/src/task/processor.rs b/ostd/src/task/processor.rs index 0659c16f6..c6dfb9932 100644 --- a/ostd/src/task/processor.rs +++ b/ostd/src/task/processor.rs @@ -1,7 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -#![allow(dead_code)] - use alloc::sync::Arc; use core::{ cell::RefCell, @@ -13,7 +11,7 @@ use super::{ task::{context_switch, TaskContext}, Task, TaskStatus, }; -use crate::{cpu_local, CpuLocal}; +use crate::cpu_local; pub struct Processor { current: Option>, @@ -49,21 +47,16 @@ cpu_local! { static PROCESSOR: RefCell = RefCell::new(Processor::new()); } -pub fn take_current_task() -> Option> { - CpuLocal::borrow_with(&PROCESSOR, |processor| { - processor.borrow_mut().take_current() - }) -} - /// Retrieves the current task running on the processor. pub fn current_task() -> Option> { - CpuLocal::borrow_with(&PROCESSOR, |processor| processor.borrow().current()) + PROCESSOR.borrow_irq_disabled().borrow().current() } pub(crate) fn get_idle_task_ctx_ptr() -> *mut TaskContext { - CpuLocal::borrow_with(&PROCESSOR, |processor| { - processor.borrow_mut().get_idle_task_ctx_ptr() - }) + PROCESSOR + .borrow_irq_disabled() + .borrow_mut() + .get_idle_task_ctx_ptr() } /// Calls this function to switch to other task by using GLOBAL_SCHEDULER @@ -134,15 +127,16 @@ fn switch_to_task(next_task: Arc) { } // Change the current task to the next task. - CpuLocal::borrow_with(&PROCESSOR, |processor| { - let mut processor = processor.borrow_mut(); + { + let processor_guard = PROCESSOR.borrow_irq_disabled(); + let mut processor = processor_guard.borrow_mut(); // We cannot directly overwrite `current` at this point. Since we are running as `current`, // we must avoid dropping `current`. Otherwise, the kernel stack may be unmapped, leading // to soundness problems. let old_current = processor.current.replace(next_task); processor.prev_task = old_current; - }); + } // SAFETY: // 1. `ctx` is only used in `schedule()`. We have exclusive access to both the current task @@ -200,7 +194,7 @@ impl PreemptInfo { #[must_use] pub struct DisablePreemptGuard { // This private field prevents user from constructing values of this type directly. - private: (), + _private: (), } impl !Send for DisablePreemptGuard {} @@ -208,7 +202,7 @@ impl !Send for DisablePreemptGuard {} impl DisablePreemptGuard { fn new() -> Self { PREEMPT_COUNT.increase_num_locks(); - Self { private: () } + Self { _private: () } } /// Transfer this guard to a new guard. diff --git a/ostd/src/trap/softirq.rs b/ostd/src/trap/softirq.rs index d5215d052..df08a08d4 100644 --- a/ostd/src/trap/softirq.rs +++ b/ostd/src/trap/softirq.rs @@ -9,7 +9,7 @@ use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use spin::Once; -use crate::{cpu_local, task::disable_preempt, CpuLocal}; +use crate::{cpu_local, task::disable_preempt}; /// A representation of a software interrupt (softirq) line. /// @@ -70,9 +70,7 @@ impl SoftIrqLine { /// /// If this line is not enabled yet, the method has no effect. pub fn raise(&self) { - CpuLocal::borrow_with(&PENDING_MASK, |mask| { - mask.fetch_or(1 << self.id, Ordering::Release); - }); + PENDING_MASK.fetch_or(1 << self.id, Ordering::Release); } /// Enables a softirq line by registering its callback. @@ -114,21 +112,17 @@ cpu_local! { /// Enables softirq in current processor. fn enable_softirq_local() { - CpuLocal::borrow_with(&IS_ENABLED, |is_enabled| { - is_enabled.store(true, Ordering::Release) - }) + IS_ENABLED.store(true, Ordering::Release); } /// Disables softirq in current processor. fn disable_softirq_local() { - CpuLocal::borrow_with(&IS_ENABLED, |is_enabled| { - is_enabled.store(false, Ordering::Release) - }) + IS_ENABLED.store(false, Ordering::Release); } /// Checks whether the softirq is enabled in current processor. fn is_softirq_enabled() -> bool { - CpuLocal::borrow_with(&IS_ENABLED, |is_enabled| is_enabled.load(Ordering::Acquire)) + IS_ENABLED.load(Ordering::Acquire) } /// Processes pending softirqs. @@ -145,23 +139,21 @@ pub(crate) fn process_pending() { let preempt_guard = disable_preempt(); disable_softirq_local(); - CpuLocal::borrow_with(&PENDING_MASK, |mask| { - for i in 0..SOFTIRQ_RUN_TIMES { - // will not reactive in this handling. - let mut action_mask = { - let pending_mask = mask.fetch_and(0, Ordering::Acquire); - pending_mask & ENABLED_MASK.load(Ordering::Acquire) - }; + for i in 0..SOFTIRQ_RUN_TIMES { + let mut action_mask = { + let pending_mask = PENDING_MASK.fetch_and(0, Ordering::Acquire); + pending_mask & ENABLED_MASK.load(Ordering::Acquire) + }; - if action_mask == 0 { - return; - } - while action_mask > 0 { - let action_id = u8::trailing_zeros(action_mask) as u8; - SoftIrqLine::get(action_id).callback.get().unwrap()(); - action_mask &= action_mask - 1; - } + if action_mask == 0 { + break; } - }); + while action_mask > 0 { + let action_id = u8::trailing_zeros(action_mask) as u8; + SoftIrqLine::get(action_id).callback.get().unwrap()(); + action_mask &= action_mask - 1; + } + } + enable_softirq_local(); }