From eee11fa813676b3c9b512ebe2f285151025832f1 Mon Sep 17 00:00:00 2001 From: Zhang Junyang Date: Wed, 13 Nov 2024 21:07:22 +0800 Subject: [PATCH] Add a read-copy-update method to RCU and allow nullable RCU --- ostd/src/sync/mod.rs | 2 +- ostd/src/sync/rcu/mod.rs | 288 ++++++++++++++++++++++++--------- ostd/src/sync/rcu/owner_ptr.rs | 15 ++ 3 files changed, 231 insertions(+), 74 deletions(-) diff --git a/ostd/src/sync/mod.rs b/ostd/src/sync/mod.rs index 7c417ec96..0ce178e47 100644 --- a/ostd/src/sync/mod.rs +++ b/ostd/src/sync/mod.rs @@ -15,7 +15,7 @@ pub(crate) use self::{guard::GuardTransfer, rcu::finish_grace_period}; pub use self::{ guard::{LocalIrqDisabled, PreemptDisabled, WriteIrqDisabled}, mutex::{ArcMutexGuard, Mutex, MutexGuard}, - rcu::{OwnerPtr, Rcu, RcuReadGuard}, + rcu::{OwnerPtr, Rcu, RcuOption, RcuReadGuard}, rwarc::{RoArc, RwArc}, rwlock::{ ArcRwLockReadGuard, ArcRwLockUpgradeableGuard, ArcRwLockWriteGuard, RwLock, diff --git a/ostd/src/sync/rcu/mod.rs b/ostd/src/sync/rcu/mod.rs index 330124f9a..e94f4f29f 100644 --- a/ostd/src/sync/rcu/mod.rs +++ b/ostd/src/sync/rcu/mod.rs @@ -5,6 +5,7 @@ use core::{ marker::PhantomData, ops::Deref, + ptr::NonNull, sync::atomic::{ AtomicPtr, Ordering::{AcqRel, Acquire}, @@ -21,122 +22,263 @@ mod owner_ptr; pub use owner_ptr::OwnerPtr; -/// Read-Copy Update Synchronization Mechanism +/// A Read-Copy Update (RCU) cell for sharing a pointer between threads. +/// +/// The pointer should be a owning pointer with type `P`, which implements +/// [`OwnerPtr`]. For example, `P` can be `Box` or `Arc`. /// /// # Overview /// -/// RCU avoids the use of lock primitives lock primitives while multiple threads -/// concurrently read and update elements that are linked through pointers and that -/// belong to shared data structures. -/// -/// Whenever a thread is inserting or deleting elements of data structures in shared -/// memory, all readers are guaranteed to see and traverse either the older or the -/// new structure, therefore avoiding inconsistencies and allowing readers to not be -/// blocked by writers. -/// -/// The type parameter `P` represents the data that this rcu is protecting. The type -/// parameter `P` must implement [`OwnerPtr`]. -/// -/// # Usage -/// -/// It is used when performance of reads is crucial and is an example of space–time -/// tradeoff, enabling fast operations at the cost of more space. -/// -/// Use [`Rcu`] in scenarios that require frequent reads and infrequent -/// updates (read-mostly). -/// -/// Use [`Rcu`] in scenarios that require high real-time reading. -/// -/// Rcu should not to be used in the scenarios that write-mostly and which need -/// consistent data. +/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high- +/// performance, low-latency read operations in concurrent systems. It allows +/// multiple readers to access shared data simultaneously without contention, +/// while writers can update the data safely in a way that does not disrupt +/// ongoing reads. RCU is particularly suited for situations where reads are +/// far more frequent than writes. +/// +/// The original design and implementation of RCU is described in paper _The +/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared- +/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal +/// 47.2 (2008). /// /// # Examples /// /// ``` -/// use aster_frame::sync::{Rcu, RcuReadGuard, RcuReclaimer}; +/// use ostd::sync::Rcu; /// /// let rcu = Rcu::new(Box::new(42)); /// -/// // Read the data protected by rcu +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&42)); +/// +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = rcu.read(); +/// +/// assert_eq!(*rcu_guard, Some(&43)); +/// ``` +#[repr(transparent)] +pub struct Rcu { + ptr: AtomicPtr<

::Target>, + // We want to implement Send and Sync explicitly. + // Having a pointer field prevents them from being implemented + // automatically by the compiler. + _marker: PhantomData<*const P::Target>, +} + +/// A Read-Copy Update (RCU) cell for sharing a _nullable_ pointer. +/// +/// This is a variant of [`Rcu`] that allows the contained pointer to be null. +/// So that it can implement `Rcu>` where `P` is not a nullable +/// pointer. It is the same as [`Rcu`] in other aspects. +/// +/// # Examples +/// +/// ``` +/// use ostd::sync::RcuOption; +/// +/// // Also allows lazy initialization. +/// static RCU: RcuOption> = RcuOption::new_none(); +/// +/// // Not initialized yet. +/// assert!(RCU.read().try_get().is_none()); +/// +/// // Initialize the data protected by RCU. +/// RCU.update(Box::new(42)); +/// +/// // Read the data protected by RCU /// { -/// let rcu_guard = rcu.get(); +/// let rcu_guard = RCU.read().try_get().unwrap(); /// assert_eq!(*rcu_guard, 42); /// } /// -/// // Update the data protected by rcu +/// // Update the data protected by RCU /// { -/// let reclaimer = rcu.replace(Box::new(43)); +/// let rcu_guard = RCU.read().try_get().unwrap(); /// -/// let rcu_guard = rcu.get(); +/// rcu_guard.compare_exchange(Box::new(43)).unwrap(); +/// +/// let rcu_guard = RCU.read().try_get().unwrap(); /// assert_eq!(*rcu_guard, 43); /// } /// ``` -pub struct Rcu { - ptr: AtomicPtr<

::Target>, - marker: PhantomData, +pub type RcuOption

= Rcu; + +// SAFETY: It is apparent that if `P::Target` is `Send`, then `Rcu

` is `Send`. +unsafe impl Send for Rcu where +

::Target: Send +{ } -impl Rcu

{ - /// Creates a new instance of Rcu with the given pointer. - pub fn new(ptr: P) -> Self { - let ptr = AtomicPtr::new(OwnerPtr::into_raw(ptr) as *mut _); +// SAFETY: To implement `Sync` for `Rcu

`, we need to meet two conditions: +// 1. `P::Target` must be `Sync` because `Rcu::get` allows concurrent access. +// 2. `P::Target` must be `Send` because `Rcu::update` may obtain an object +// of `P` created on another thread. +unsafe impl Sync for Rcu where +

::Target: Send + Sync +{ +} + +// Non-nullable RCU cell. +impl Rcu { + /// Creates a new RCU cell with the given pointer. + pub fn new(pointer: P) -> Self { + let ptr =

::into_raw(pointer).cast_mut(); + let ptr = AtomicPtr::new(ptr); Self { ptr, - marker: PhantomData, + _marker: PhantomData, + } + } +} + +// Nullable RCU cell. +impl Rcu { + /// Creates a new uninitialized RCU cell. + /// + /// Initialization can be done by calling + /// [`RcuReadGuard::compare_exchange`] after getting a read + /// guard using [`Rcu::read`]. Then only the first initialization will be + /// successful. If initialization can be done multiple times, using + /// [`Rcu::update`] is fine. + pub const fn new_none() -> Self { + let ptr = AtomicPtr::new(core::ptr::null_mut()); + Self { + ptr, + _marker: PhantomData, + } + } +} + +impl Rcu { + /// Replaces the current pointer with a new pointer. + /// + /// This function updates the pointer to the new pointer regardless of the + /// original pointer. If the original pointer is not NULL, it will be + /// dropped after the grace period. + /// + /// Oftentimes this function is not recommended unless you have + /// synchronized writes with locks. Otherwise, you can use [`Self::read`] + /// and then [`RcuReadGuard::compare_exchange`] to update the pointer. + pub fn update(&self, new_ptr: P) { + let new_ptr =

::into_raw(new_ptr).cast_mut(); + let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel); + + if let Some(p) = NonNull::new(old_raw_ptr) { + // SAFETY: It was previously returned by `into_raw`. + unsafe { delay_drop::

(p) }; } } - /// Retrieves a read guard for the RCU mechanism. + /// Retrieves a read guard for the RCU cell. /// - /// This method returns a `RcuReadGuard` which allows read-only access to the - /// underlying data protected by the RCU mechanism. - pub fn get(&self) -> RcuReadGuard<'_, P> { + /// The guard allows read-only access to the data protected by RCU. + /// + /// If the RCU cell is nullable, the guard will be nullable and you can + /// only dereference it after checking with [`RcuReadGuard::try_get`]. + /// If the RCU cell is non-nullable, the guard will be non-nullable and + /// you can dereference it directly. + pub fn read(&self) -> RcuReadGuard<'_, P, NULLABLE> { let guard = disable_preempt(); - let obj = unsafe { &*self.ptr.load(Acquire) }; RcuReadGuard { - obj, - _rcu: self, + obj_ptr: self.ptr.load(Acquire), + rcu: self, _inner_guard: guard, } } } -impl Rcu

{ - /// Replaces the current pointer with a new pointer. - pub fn replace(&self, new_ptr: P) { - let new_ptr =

::into_raw(new_ptr) as *mut _; - let old_ptr = { - let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel); - // SAFETY: It is valid because it was previously returned by `into_raw`. - unsafe {

::from_raw(old_raw_ptr) } - }; - - let rcu_monitor = RCU_MONITOR.get().unwrap(); - rcu_monitor.after_grace_period(move || { - drop(old_ptr); - }); - } -} - -/// A guard that allows read-only access to the data protected by the RCU -/// mechanism. -/// -/// Note that the data read can be outdated if the data is updated by another -/// task after acquiring the guard. -pub struct RcuReadGuard<'a, P: OwnerPtr> { - obj: &'a

::Target, - _rcu: &'a Rcu

, +/// A guard that allows read-only access to the initialized data protected +/// by the RCU mechanism. +pub struct RcuReadGuard<'a, P: OwnerPtr, const NULLABLE: bool> { + /// If maybe uninitialized, the pointer can be NULL. + obj_ptr: *mut

::Target, + rcu: &'a Rcu, _inner_guard: DisabledPreemptGuard, } -impl Deref for RcuReadGuard<'_, P> { +// Non-nullable RCU guard can be directly dereferenced. +impl Deref for RcuReadGuard<'_, P, false> { type Target =

::Target; fn deref(&self) -> &Self::Target { - self.obj + // SAFETY: Since the preemption is disabled, the pointer is valid + // because other writers won't release the allocation until this task + // passes the quiescent state. + // And this pointer is not NULL. + unsafe { &*self.obj_ptr } } } +// Nullable RCU guard can be dereferenced after checking. +impl<'a, P: OwnerPtr> RcuReadGuard<'a, P, true> { + /// Tries to get the initialized read guard. + /// + /// If the RCU cell is not initialized, this function will return + /// [`Err`] with the guard itself unchanged. Otherwise a dereferenceable + /// read guard will be returned. + pub fn try_get(self) -> Result, Self> { + if self.obj_ptr.is_null() { + return Err(self); + } + Ok(RcuReadGuard { + obj_ptr: self.obj_ptr, + // SAFETY: It is initialized. The layout is the same. + rcu: unsafe { core::mem::transmute::<&Rcu, &Rcu>(self.rcu) }, + _inner_guard: self._inner_guard, + }) + } +} + +impl RcuReadGuard<'_, P, NULLABLE> { + /// Tries to replace the already read pointer with a new pointer. + /// + /// If another thread has updated the pointer after the read, this + /// function will fail and return the new pointer. Otherwise, it will + /// replace the pointer with the new one and drop the old pointer after + /// the grace period. + /// + /// If spinning on this function, it is recommended to relax the CPU + /// or yield the task on failure. Otherwise contention will occur. + /// + /// This API does not help to avoid + /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem). + pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> { + let new_ptr =

::into_raw(new_ptr).cast_mut(); + + if self + .rcu + .ptr + .compare_exchange(self.obj_ptr, new_ptr, AcqRel, Acquire) + .is_err() + { + // SAFETY: It was previously returned by `into_raw`. + return Err(unsafe {

::from_raw(new_ptr) }); + } + + if let Some(p) = NonNull::new(self.obj_ptr) { + // SAFETY: It was previously returned by `into_raw`. + unsafe { delay_drop::

(p) }; + } + + Ok(()) + } +} + +/// # Safety +/// +/// The pointer must be previously returned by `into_raw` and the pointer +/// must be only be dropped once. +unsafe fn delay_drop(pointer: NonNull<

::Target>) { + // SAFETY: The pointer is not NULL. + let p = unsafe {

::from_raw(pointer.as_ptr().cast_const()) }; + let rcu_monitor = RCU_MONITOR.get().unwrap(); + rcu_monitor.after_grace_period(move || { + drop(p); + }); +} + /// Finishes the current grace period. /// /// This function is called when the current grace period on current CPU is diff --git a/ostd/src/sync/rcu/owner_ptr.rs b/ostd/src/sync/rcu/owner_ptr.rs index 88c11f301..5439d9e91 100644 --- a/ostd/src/sync/rcu/owner_ptr.rs +++ b/ostd/src/sync/rcu/owner_ptr.rs @@ -13,6 +13,9 @@ pub trait OwnerPtr: 'static { // TODO: allow ?Sized type Target; + /// Creates a new pointer with the given value. + fn new(value: Self::Target) -> Self; + /// Converts to a raw pointer. /// /// If `Self` owns the object that it refers to (e.g., `Box<_>`), then @@ -31,6 +34,10 @@ pub trait OwnerPtr: 'static { impl OwnerPtr for Box { type Target = T; + fn new(value: Self::Target) -> Self { + Box::new(value) + } + fn into_raw(self) -> *const Self::Target { Box::into_raw(self) as *const _ } @@ -43,6 +50,10 @@ impl OwnerPtr for Box { impl OwnerPtr for Arc { type Target = T; + fn new(value: Self::Target) -> Self { + Arc::new(value) + } + fn into_raw(self) -> *const Self::Target { Arc::into_raw(self) } @@ -63,6 +74,10 @@ where { type Target = P::Target; + fn new(value: Self::Target) -> Self { + Some(P::new(value)) + } + fn into_raw(self) -> *const Self::Target { self.map(|p|

::into_raw(p)) .unwrap_or(core::ptr::null())