diff --git a/osdk/deps/frame-allocator/src/pools/balancing.rs b/osdk/deps/frame-allocator/src/pools/balancing.rs index 400e3481..e62a9dd7 100644 --- a/osdk/deps/frame-allocator/src/pools/balancing.rs +++ b/osdk/deps/frame-allocator/src/pools/balancing.rs @@ -2,13 +2,9 @@ //! Controlling the balancing between CPU-local free pools and the global free pool. -use core::sync::atomic::Ordering; - use ostd::cpu::num_cpus; -use super::{ - lesser_order_of, BuddyOrder, BuddySet, GLOBAL_POOL, GLOBAL_POOL_SIZE, MAX_LOCAL_BUDDY_ORDER, -}; +use super::{lesser_order_of, BuddyOrder, BuddySet, OnDemandGlobalLock, MAX_LOCAL_BUDDY_ORDER}; use crate::chunk::split_to_order; @@ -50,8 +46,8 @@ fn cache_maximal_size(global_size: usize) -> usize { } /// Balances a local cache and the global free pool. -pub fn balance(local: &mut BuddySet) { - let global_size = GLOBAL_POOL_SIZE.load(Ordering::Relaxed); +pub fn balance(local: &mut BuddySet, global: &mut OnDemandGlobalLock) { + let global_size = global.get_global_size(); let minimal_local_size = cache_minimal_size(global_size); let expected_local_size = cache_expected_size(global_size); @@ -67,11 +63,8 @@ pub fn balance(local: &mut BuddySet) { let expected_removal = local_size - expected_local_size; let lesser_order = lesser_order_of(expected_removal); - let mut global_pool_lock = GLOBAL_POOL.lock(); - balance_to(local, &mut *global_pool_lock, lesser_order); - - GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); + balance_to(local, &mut *global.get(), lesser_order); } else if local_size < minimal_local_size { // Move global frames to the local pool. if global_size == 0 { @@ -80,11 +73,8 @@ pub fn balance(local: &mut BuddySet) { let expected_allocation = expected_local_size - local_size; let lesser_order = lesser_order_of(expected_allocation); - let mut global_pool_lock = GLOBAL_POOL.lock(); - balance_to(&mut *global_pool_lock, local, lesser_order); - - GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); + balance_to(&mut *global.get(), local, lesser_order); } } diff --git a/osdk/deps/frame-allocator/src/pools/mod.rs b/osdk/deps/frame-allocator/src/pools/mod.rs index 1bb2123b..90d0b29e 100644 --- a/osdk/deps/frame-allocator/src/pools/mod.rs +++ b/osdk/deps/frame-allocator/src/pools/mod.rs @@ -10,10 +10,9 @@ use core::{ }; use ostd::{ - cpu::PinCurrentCpu, cpu_local, mm::Paddr, - sync::{LocalIrqDisabled, SpinLock}, + sync::{LocalIrqDisabled, SpinLock, SpinLockGuard}, trap::DisabledLocalIrqGuard, }; @@ -24,12 +23,12 @@ use super::set::BuddySet; /// The global free buddies. static GLOBAL_POOL: SpinLock, LocalIrqDisabled> = SpinLock::new(BuddySet::new_empty()); +/// A snapshot of the total size of the global free buddies, not precise. static GLOBAL_POOL_SIZE: AtomicUsize = AtomicUsize::new(0); // CPU-local free buddies. cpu_local! { static LOCAL_POOL: RefCell> = RefCell::new(BuddySet::new_empty()); - static LOCAL_POOL_SIZE: AtomicUsize = AtomicUsize::new(0); } /// Maximum supported order of the buddy system. @@ -55,6 +54,7 @@ const MAX_LOCAL_BUDDY_ORDER: BuddyOrder = 18; pub(super) fn alloc(guard: &DisabledLocalIrqGuard, layout: Layout) -> Option { let local_pool_cell = LOCAL_POOL.get_with(guard); let mut local_pool = local_pool_cell.borrow_mut(); + let mut global_pool = OnDemandGlobalLock::new(); let size_order = greater_order_of(layout.size()); let align_order = greater_order_of(layout.align()); @@ -68,7 +68,7 @@ pub(super) fn alloc(guard: &DisabledLocalIrqGuard, layout: Layout) -> Option Option layout.size() { if let Some(chunk_addr) = chunk_addr { - add_free_memory_to( + do_dealloc( &mut local_pool, - guard, + &mut global_pool, chunk_addr + layout.size(), allocated_size - layout.size(), ); } } - balancing::balance(local_pool.deref_mut()); + balancing::balance(local_pool.deref_mut(), &mut global_pool); - LOCAL_POOL_SIZE - .get_on_cpu(guard.current_cpu()) - .store(local_pool.total_size(), Ordering::Relaxed); + global_pool.update_global_size_if_locked(); chunk_addr } @@ -100,46 +98,76 @@ pub(super) fn alloc(guard: &DisabledLocalIrqGuard, layout: Layout) -> Option, - guard: &DisabledLocalIrqGuard, + global_pool: &mut OnDemandGlobalLock, addr: Paddr, size: usize, ) { split_to_chunks(addr, size).for_each(|(addr, order)| { if order >= MAX_LOCAL_BUDDY_ORDER { - dealloc_to_global_pool(addr, order); + global_pool.get().insert_chunk(addr, order); } else { local_pool.insert_chunk(addr, order); } }); - - balancing::balance(local_pool); - LOCAL_POOL_SIZE - .get_on_cpu(guard.current_cpu()) - .store(local_pool.total_size(), Ordering::Relaxed); } -fn alloc_from_global_pool(order: BuddyOrder) -> Option { - let mut lock_guard = GLOBAL_POOL.lock(); - let res = lock_guard.alloc_chunk(order); - GLOBAL_POOL_SIZE.store(lock_guard.total_size(), Ordering::Relaxed); - res +type GlobalLockGuard = SpinLockGuard<'static, BuddySet, LocalIrqDisabled>; + +/// An on-demand guard that locks the global pool when needed. +/// +/// It helps to avoid unnecessarily locking the global pool, and also avoids +/// repeatedly locking the global pool when it is needed multiple times. +struct OnDemandGlobalLock { + guard: Option, } -fn dealloc_to_global_pool(addr: Paddr, order: BuddyOrder) { - let mut lock_guard = GLOBAL_POOL.lock(); - lock_guard.insert_chunk(addr, order); - GLOBAL_POOL_SIZE.store(lock_guard.total_size(), Ordering::Relaxed); +impl OnDemandGlobalLock { + fn new() -> Self { + Self { guard: None } + } + + fn get(&mut self) -> &mut GlobalLockGuard { + self.guard.get_or_insert_with(|| GLOBAL_POOL.lock()) + } + + /// Updates [`GLOBAL_POOL_SIZE`] if the global pool is locked. + fn update_global_size_if_locked(&self) { + if let Some(guard) = self.guard.as_ref() { + GLOBAL_POOL_SIZE.store(guard.total_size(), Ordering::Relaxed); + } + } + + /// Returns the size of the global pool. + /// + /// If the global pool is locked, returns the actual size of the global pool. + /// Otherwise, returns the last snapshot of the global pool size by loading + /// [`GLOBAL_POOL_SIZE`]. + fn get_global_size(&self) -> usize { + if let Some(guard) = self.guard.as_ref() { + guard.total_size() + } else { + GLOBAL_POOL_SIZE.load(Ordering::Relaxed) + } + } }