diff --git a/osdk/deps/frame-allocator/src/cache.rs b/osdk/deps/frame-allocator/src/cache.rs index 3942e511a..7a3957b01 100644 --- a/osdk/deps/frame-allocator/src/cache.rs +++ b/osdk/deps/frame-allocator/src/cache.rs @@ -69,14 +69,14 @@ impl CacheArray Option 4 { - super::pools::add_free_memory(guard, addr, size); + super::pools::dealloc(guard, addr, size); return; } @@ -142,10 +142,10 @@ pub(super) fn add_free_memory(guard: &DisabledLocalIrqGuard, addr: Paddr, size: let mut cache = cache_cell.borrow_mut(); match nr_frames { - 1 => cache.cache1.add_free_memory(guard, addr), - 2 => cache.cache2.add_free_memory(guard, addr), - 3 => cache.cache3.add_free_memory(guard, addr), - 4 => cache.cache4.add_free_memory(guard, addr), - _ => super::pools::add_free_memory(guard, addr, size), + 1 => cache.cache1.dealloc(guard, addr), + 2 => cache.cache2.dealloc(guard, addr), + 3 => cache.cache3.dealloc(guard, addr), + 4 => cache.cache4.dealloc(guard, addr), + _ => super::pools::dealloc(guard, addr, size), } } diff --git a/osdk/deps/frame-allocator/src/lib.rs b/osdk/deps/frame-allocator/src/lib.rs index 970c0b66b..6fc62188d 100644 --- a/osdk/deps/frame-allocator/src/lib.rs +++ b/osdk/deps/frame-allocator/src/lib.rs @@ -61,12 +61,14 @@ impl GlobalFrameAllocator for FrameAllocator { } fn dealloc(&self, addr: Paddr, size: usize) { - self.add_free_memory(addr, size); + let guard = trap::disable_local(); + per_cpu_counter::add_free_size(&guard, size); + cache::dealloc(&guard, addr, size); } fn add_free_memory(&self, addr: Paddr, size: usize) { let guard = trap::disable_local(); per_cpu_counter::add_free_size(&guard, size); - cache::add_free_memory(&guard, addr, size); + pools::add_free_memory(&guard, addr, size); } } diff --git a/osdk/deps/frame-allocator/src/pools/balancing.rs b/osdk/deps/frame-allocator/src/pools/balancing.rs new file mode 100644 index 000000000..d5f7e8576 --- /dev/null +++ b/osdk/deps/frame-allocator/src/pools/balancing.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: MPL-2.0 + +//! Controlling the balancing between CPU-local free pools and the global free pool. + +use core::sync::atomic::Ordering; + +use ostd::cpu::num_cpus; + +use super::{ + lesser_order_of, BuddyOrder, BuddySet, GLOBAL_POOL, GLOBAL_POOL_SIZE, MAX_LOCAL_BUDDY_ORDER, +}; + +use crate::chunk::size_of_order; + +/// Controls the expected size of cache for each CPU-local free pool. +/// +/// The expected size will be the size of `GLOBAL_POOL` divided by the number +/// of the CPUs, and then divided by this constant. +const CACHE_EXPECTED_PORTION: usize = 2; + +/// Returns the expected size of cache for each CPU-local free pool. +/// +/// It depends on the size of the global free pool. +fn cache_expected_size(global_size: usize) -> usize { + global_size / num_cpus() / CACHE_EXPECTED_PORTION +} + +/// Controls the minimal size of cache for each CPU-local free pool. +/// +/// The minimal will be the expected size divided by this constant. +const CACHE_MINIMAL_PORTION: usize = 8; + +/// Returns the minimal size of cache for each CPU-local free pool. +/// +/// It depends on the size of the global free pool. +fn cache_minimal_size(global_size: usize) -> usize { + cache_expected_size(global_size) / CACHE_MINIMAL_PORTION +} + +/// Controls the maximal size of cache for each CPU-local free pool. +/// +/// The maximal will be the expected size multiplied by this constant. +const CACHE_MAXIMAL_MULTIPLIER: usize = 2; + +/// Returns the maximal size of cache for each CPU-local free pool. +/// +/// It depends on the size of the global free pool. +fn cache_maximal_size(global_size: usize) -> usize { + cache_expected_size(global_size) * CACHE_MAXIMAL_MULTIPLIER +} + +/// Balances a local cache and the global free pool. +pub fn balance(local: &mut BuddySet) { + let global_size = GLOBAL_POOL_SIZE.load(Ordering::Relaxed); + + let minimal_local_size = cache_minimal_size(global_size); + let expected_local_size = cache_expected_size(global_size); + let maximal_local_size = cache_maximal_size(global_size); + + let local_size = local.total_size(); + + if local_size >= maximal_local_size { + // Move local frames to the global pool. + if local_size == 0 { + return; + } + + let expected_removal = local_size - expected_local_size; + let lesser_order = lesser_order_of(expected_removal); + let mut global_pool_lock = GLOBAL_POOL.lock(); + + balance_to(local, &mut *global_pool_lock, lesser_order); + + GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); + } else if local_size < minimal_local_size { + // Move global frames to the local pool. + if global_size == 0 { + return; + } + + let expected_allocation = expected_local_size - local_size; + let lesser_order = lesser_order_of(expected_allocation); + let mut global_pool_lock = GLOBAL_POOL.lock(); + + balance_to(&mut *global_pool_lock, local, lesser_order); + + GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); + } +} + +/// Balances from `a` to `b`. +fn balance_to( + a: &mut BuddySet, + b: &mut BuddySet, + order: BuddyOrder, +) { + let allocated_from_a = a.alloc_chunk(order); + + if let Some(addr) = allocated_from_a { + if order >= MAX_ORDER2 { + let inserted_order = MAX_ORDER2 - 1; + for i in 0..(1 << (order - inserted_order)) as usize { + let split_addr = addr + size_of_order(inserted_order) * i; + b.insert_chunk(split_addr, inserted_order); + } + } else { + b.insert_chunk(addr, order); + } + } else { + // Maybe the chunk size is too large. + // Try to reduce the order and balance again. + if order > 1 { + balance_to(a, b, order - 1); + balance_to(a, b, order - 1); + } + } +} diff --git a/osdk/deps/frame-allocator/src/pools/mod.rs b/osdk/deps/frame-allocator/src/pools/mod.rs index b637cab8b..da171be05 100644 --- a/osdk/deps/frame-allocator/src/pools/mod.rs +++ b/osdk/deps/frame-allocator/src/pools/mod.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 +mod balancing; + use core::{ alloc::Layout, cell::RefCell, @@ -56,8 +58,8 @@ pub(super) fn alloc(guard: &DisabledLocalIrqGuard, layout: Layout) -> Option Option layout.size() { if let Some(chunk_addr) = chunk_addr { - dealloc_in( + add_free_memory_to( &mut local_pool, guard, chunk_addr + layout.size(), allocated_size - layout.size(), ); } - } else { - balancing::balance(local_pool.deref_mut()); } + balancing::balance(local_pool.deref_mut()); + LOCAL_POOL_SIZE .get_on_cpu(guard.current_cpu()) .store(local_pool.total_size(), Ordering::Relaxed); @@ -95,14 +97,21 @@ pub(super) fn alloc(guard: &DisabledLocalIrqGuard, layout: Layout) -> Option, guard: &DisabledLocalIrqGuard, mut addr: Paddr, @@ -141,121 +150,3 @@ fn dealloc_to_global_pool(addr: Paddr, order: BuddyOrder) { lock_guard.insert_chunk(addr, order); GLOBAL_POOL_SIZE.store(lock_guard.total_size(), Ordering::Relaxed); } - -pub mod balancing { - //! Controlling the balancing between CPU-local free pools and the global free pool. - - use core::sync::atomic::Ordering; - - use ostd::cpu::num_cpus; - - use super::{ - lesser_order_of, BuddyOrder, BuddySet, GLOBAL_POOL, GLOBAL_POOL_SIZE, MAX_LOCAL_BUDDY_ORDER, - }; - - use crate::chunk::size_of_order; - - /// Controls the expected size of cache for each CPU-local free pool. - /// - /// The expected size will be the size of `GLOBAL_POOL` divided by the number - /// of the CPUs, and then divided by this constant. - const CACHE_EXPECTED_PORTION: usize = 2; - - /// Returns the expected size of cache for each CPU-local free pool. - /// - /// It depends on the size of the global free pool. - fn cache_expected_size(global_size: usize) -> usize { - global_size / num_cpus() / CACHE_EXPECTED_PORTION - } - - /// Controls the minimal size of cache for each CPU-local free pool. - /// - /// The minimal will be the expected size divided by this constant. - const CACHE_MINIMAL_PORTION: usize = 8; - - /// Returns the minimal size of cache for each CPU-local free pool. - /// - /// It depends on the size of the global free pool. - fn cache_minimal_size(global_size: usize) -> usize { - cache_expected_size(global_size) / CACHE_MINIMAL_PORTION - } - - /// Controls the maximal size of cache for each CPU-local free pool. - /// - /// The maximal will be the expected size multiplied by this constant. - const CACHE_MAXIMAL_MULTIPLIER: usize = 2; - - /// Returns the maximal size of cache for each CPU-local free pool. - /// - /// It depends on the size of the global free pool. - fn cache_maximal_size(global_size: usize) -> usize { - cache_expected_size(global_size) * CACHE_MAXIMAL_MULTIPLIER - } - - /// Balances a local cache and the global free pool. - pub fn balance(local: &mut BuddySet) { - let global_size = GLOBAL_POOL_SIZE.load(Ordering::Relaxed); - - let minimal_local_size = cache_minimal_size(global_size); - let expected_local_size = cache_expected_size(global_size); - let maximal_local_size = cache_maximal_size(global_size); - - let local_size = local.total_size(); - - if local_size >= maximal_local_size { - // Move local frames to the global pool. - if local_size == 0 { - return; - } - - let expected_removal = local_size - expected_local_size; - let lesser_order = lesser_order_of(expected_removal); - let mut global_pool_lock = GLOBAL_POOL.lock(); - - balance_to(local, &mut *global_pool_lock, lesser_order); - - GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); - } else if local_size < minimal_local_size { - // Move global frames to the local pool. - if global_size == 0 { - return; - } - - let expected_allocation = expected_local_size - local_size; - let lesser_order = lesser_order_of(expected_allocation); - let mut global_pool_lock = GLOBAL_POOL.lock(); - - balance_to(&mut *global_pool_lock, local, lesser_order); - - GLOBAL_POOL_SIZE.store(global_pool_lock.total_size(), Ordering::Relaxed); - } - } - - /// Balances from `a` to `b`. - fn balance_to( - a: &mut BuddySet, - b: &mut BuddySet, - order: BuddyOrder, - ) { - let allocated_from_a = a.alloc_chunk(order); - - if let Some(addr) = allocated_from_a { - if order >= MAX_ORDER2 { - let inserted_order = MAX_ORDER2 - 1; - for i in 0..(1 << (order - inserted_order)) as usize { - let split_addr = addr + size_of_order(inserted_order) * i; - b.insert_chunk(split_addr, inserted_order); - } - } else { - b.insert_chunk(addr, order); - } - } else { - // Maybe the chunk size is too large. - // Try to reduce the order and balance again. - if order > 1 { - balance_to(a, b, order - 1); - balance_to(a, b, order - 1); - } - } - } -}