Add a read-copy-update method to RCU and allow nullable RCU

ostd/src/sync/mod.rs

@@ -15,7 +15,7 @@ pub(crate) use self::{guard::GuardTransfer, rcu::finish_grace_period};
 pub use self::{
     guard::{LocalIrqDisabled, PreemptDisabled, WriteIrqDisabled},
     mutex::{ArcMutexGuard, Mutex, MutexGuard},
-    rcu::{OwnerPtr, Rcu, RcuReadGuard},
+    rcu::{OwnerPtr, Rcu, RcuOption, RcuReadGuard},
     rwarc::{RoArc, RwArc},
     rwlock::{
         ArcRwLockReadGuard, ArcRwLockUpgradeableGuard, ArcRwLockWriteGuard, RwLock,

ostd/src/sync/rcu/mod.rs

@@ -5,6 +5,7 @@
 use core::{
     marker::PhantomData,
     ops::Deref,
+    ptr::NonNull,
     sync::atomic::{
         AtomicPtr,
         Ordering::{AcqRel, Acquire},
@@ -21,122 +22,263 @@ mod owner_ptr;
 
 pub use owner_ptr::OwnerPtr;
 
-/// Read-Copy Update Synchronization Mechanism
+/// A Read-Copy Update (RCU) cell for sharing a pointer between threads.
+///
+/// The pointer should be a owning pointer with type `P`, which implements
+/// [`OwnerPtr`]. For example, `P` can be `Box<T>` or `Arc<T>`.
 ///
 /// # Overview
 ///
-/// RCU avoids the use of lock primitives lock primitives while multiple threads
+/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high-
-/// concurrently read and update elements that are linked through pointers and that
+/// performance, low-latency read operations in concurrent systems. It allows
-/// belong to shared data structures.
+/// multiple readers to access shared data simultaneously without contention,
-///
+/// while writers can update the data safely in a way that does not disrupt
-/// Whenever a thread is inserting or deleting elements of data structures in shared
+/// ongoing reads. RCU is particularly suited for situations where reads are
-/// memory, all readers are guaranteed to see and traverse either the older or the
+/// far more frequent than writes.  
-/// new structure, therefore avoiding inconsistencies and allowing readers to not be
+///  
-/// blocked by writers.
+/// The original design and implementation of RCU is described in paper _The
-///
+/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared-
-/// The type parameter `P` represents the data that this rcu is protecting. The type
+/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal
-/// parameter `P` must implement [`OwnerPtr`].
+/// 47.2 (2008).
-///
-/// # Usage
-///
-/// It is used when performance of reads is crucial and is an example of space–time
-/// tradeoff, enabling fast operations at the cost of more space.
-///
-/// Use [`Rcu`] in scenarios that require frequent reads and infrequent
-/// updates (read-mostly).
-///
-/// Use [`Rcu`] in scenarios that require high real-time reading.
-///
-/// Rcu should not to be used in the scenarios that write-mostly and which need
-/// consistent data.
 ///
 /// # Examples
 ///
 /// ```
-/// use aster_frame::sync::{Rcu, RcuReadGuard, RcuReclaimer};
+/// use ostd::sync::Rcu;
 ///
 /// let rcu = Rcu::new(Box::new(42));
 ///
-/// // Read the data protected by rcu
+/// let rcu_guard = rcu.read();
+///
+/// assert_eq!(*rcu_guard, Some(&42));
+///
+/// rcu_guard.compare_exchange(Box::new(43)).unwrap();
+///
+/// let rcu_guard = rcu.read();
+///
+/// assert_eq!(*rcu_guard, Some(&43));
+/// ```
+#[repr(transparent)]
+pub struct Rcu<P: OwnerPtr, const NULLABLE: bool = false> {
+    ptr: AtomicPtr<<P as OwnerPtr>::Target>,
+    // We want to implement Send and Sync explicitly.
+    // Having a pointer field prevents them from being implemented
+    // automatically by the compiler.
+    _marker: PhantomData<*const P::Target>,
+}
+
+/// A Read-Copy Update (RCU) cell for sharing a _nullable_ pointer.  
+///
+/// This is a variant of [`Rcu`] that allows the contained pointer to be null.
+/// So that it can implement `Rcu<Option<P>>` where `P` is not a nullable
+/// pointer. It is the same as [`Rcu`] in other aspects.
+///
+/// # Examples
+///
+/// ```
+/// use ostd::sync::RcuOption;
+///
+/// // Also allows lazy initialization.
+/// static RCU: RcuOption<Box<usize>> = RcuOption::new_none();
+///
+/// // Not initialized yet.
+/// assert!(RCU.read().try_get().is_none());
+///
+/// // Initialize the data protected by RCU.
+/// RCU.update(Box::new(42));
+///
+/// // Read the data protected by RCU
 /// {
-///     let rcu_guard = rcu.get();
+///     let rcu_guard = RCU.read().try_get().unwrap();
 ///     assert_eq!(*rcu_guard, 42);
 /// }
 ///
-/// // Update the data protected by rcu
+/// // Update the data protected by RCU
 /// {
-///     let reclaimer = rcu.replace(Box::new(43));
+///     let rcu_guard = RCU.read().try_get().unwrap();
 ///
-///     let rcu_guard = rcu.get();
+///     rcu_guard.compare_exchange(Box::new(43)).unwrap();
+///
+///     let rcu_guard = RCU.read().try_get().unwrap();
 ///     assert_eq!(*rcu_guard, 43);
 /// }
 /// ```
-pub struct Rcu<P: OwnerPtr> {
+pub type RcuOption<P> = Rcu<P, true>;
-    ptr: AtomicPtr<<P as OwnerPtr>::Target>,
+
-    marker: PhantomData<P::Target>,
+// SAFETY: It is apparent that if `P::Target` is `Send`, then `Rcu<P>` is `Send`.
+unsafe impl<P: OwnerPtr, const NULLABLE: bool> Send for Rcu<P, NULLABLE> where
+    <P as OwnerPtr>::Target: Send
+{
 }
 
-impl<P: OwnerPtr> Rcu<P> {
+// SAFETY: To implement `Sync` for `Rcu<P>`, we need to meet two conditions:
-    /// Creates a new instance of Rcu with the given pointer.
+//  1. `P::Target` must be `Sync` because `Rcu::get` allows concurrent access.
-    pub fn new(ptr: P) -> Self {
+//  2. `P::Target` must be `Send` because `Rcu::update` may obtain an object
-        let ptr = AtomicPtr::new(OwnerPtr::into_raw(ptr) as *mut _);
+//     of `P` created on another thread.
+unsafe impl<P: OwnerPtr, const NULLABLE: bool> Sync for Rcu<P, NULLABLE> where
+    <P as OwnerPtr>::Target: Send + Sync
+{
+}
+
+// Non-nullable RCU cell.
+impl<P: OwnerPtr> Rcu<P, false> {
+    /// Creates a new RCU cell with the given pointer.
+    pub fn new(pointer: P) -> Self {
+        let ptr = <P as OwnerPtr>::into_raw(pointer).cast_mut();
+        let ptr = AtomicPtr::new(ptr);
         Self {
             ptr,
-            marker: PhantomData,
+            _marker: PhantomData,
+        }
+    }
+}
+
+// Nullable RCU cell.
+impl<P: OwnerPtr> Rcu<P, true> {
+    /// Creates a new uninitialized RCU cell.
+    ///
+    /// Initialization can be done by calling
+    /// [`RcuReadGuard::compare_exchange`] after getting a read
+    /// guard using [`Rcu::read`]. Then only the first initialization will be
+    /// successful. If initialization can be done multiple times, using
+    /// [`Rcu::update`] is fine.
+    pub const fn new_none() -> Self {
+        let ptr = AtomicPtr::new(core::ptr::null_mut());
+        Self {
+            ptr,
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<P: OwnerPtr + Send, const NULLABLE: bool> Rcu<P, NULLABLE> {
+    /// Replaces the current pointer with a new pointer.
+    ///
+    /// This function updates the pointer to the new pointer regardless of the
+    /// original pointer. If the original pointer is not NULL, it will be
+    /// dropped after the grace period.
+    ///
+    /// Oftentimes this function is not recommended unless you have
+    /// synchronized writes with locks. Otherwise, you can use [`Self::read`]
+    /// and then [`RcuReadGuard::compare_exchange`] to update the pointer.
+    pub fn update(&self, new_ptr: P) {
+        let new_ptr = <P as OwnerPtr>::into_raw(new_ptr).cast_mut();
+        let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel);
+
+        if let Some(p) = NonNull::new(old_raw_ptr) {
+            // SAFETY: It was previously returned by `into_raw`.
+            unsafe { delay_drop::<P>(p) };
         }
     }
 
-    /// Retrieves a read guard for the RCU mechanism.
+    /// Retrieves a read guard for the RCU cell.
     ///
-    /// This method returns a `RcuReadGuard` which allows read-only access to the
+    /// The guard allows read-only access to the data protected by RCU.
-    /// underlying data protected by the RCU mechanism.
+    ///
-    pub fn get(&self) -> RcuReadGuard<'_, P> {
+    /// If the RCU cell is nullable, the guard will be nullable and you can
+    /// only dereference it after checking with [`RcuReadGuard::try_get`].
+    /// If the RCU cell is non-nullable, the guard will be non-nullable and
+    /// you can dereference it directly.
+    pub fn read(&self) -> RcuReadGuard<'_, P, NULLABLE> {
         let guard = disable_preempt();
-        let obj = unsafe { &*self.ptr.load(Acquire) };
         RcuReadGuard {
-            obj,
+            obj_ptr: self.ptr.load(Acquire),
-            _rcu: self,
+            rcu: self,
             _inner_guard: guard,
         }
     }
 }
 
-impl<P: OwnerPtr + Send> Rcu<P> {
+/// A guard that allows read-only access to the initialized data protected
-    /// Replaces the current pointer with a new pointer.
+/// by the RCU mechanism.
-    pub fn replace(&self, new_ptr: P) {
+pub struct RcuReadGuard<'a, P: OwnerPtr, const NULLABLE: bool> {
-        let new_ptr = <P as OwnerPtr>::into_raw(new_ptr) as *mut _;
+    /// If maybe uninitialized, the pointer can be NULL.
-        let old_ptr = {
+    obj_ptr: *mut <P as OwnerPtr>::Target,
-            let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel);
+    rcu: &'a Rcu<P, NULLABLE>,
-            // SAFETY: It is valid because it was previously returned by `into_raw`.
-            unsafe { <P as OwnerPtr>::from_raw(old_raw_ptr) }
-        };
-
-        let rcu_monitor = RCU_MONITOR.get().unwrap();
-        rcu_monitor.after_grace_period(move || {
-            drop(old_ptr);
-        });
-    }
-}
-
-/// A guard that allows read-only access to the data protected by the RCU
-/// mechanism.
-///
-/// Note that the data read can be outdated if the data is updated by another
-/// task after acquiring the guard.
-pub struct RcuReadGuard<'a, P: OwnerPtr> {
-    obj: &'a <P as OwnerPtr>::Target,
-    _rcu: &'a Rcu<P>,
     _inner_guard: DisabledPreemptGuard,
 }
 
-impl<P: OwnerPtr> Deref for RcuReadGuard<'_, P> {
+// Non-nullable RCU guard can be directly dereferenced.
+impl<P: OwnerPtr> Deref for RcuReadGuard<'_, P, false> {
     type Target = <P as OwnerPtr>::Target;
 
     fn deref(&self) -> &Self::Target {
-        self.obj
+        // SAFETY: Since the preemption is disabled, the pointer is valid
+        // because other writers won't release the allocation until this task
+        // passes the quiescent state.
+        // And this pointer is not NULL.
+        unsafe { &*self.obj_ptr }
     }
 }
 
+// Nullable RCU guard can be dereferenced after checking.
+impl<'a, P: OwnerPtr> RcuReadGuard<'a, P, true> {
+    /// Tries to get the initialized read guard.
+    ///
+    /// If the RCU cell is not initialized, this function will return
+    /// [`Err`] with the guard itself unchanged. Otherwise a dereferenceable
+    /// read guard will be returned.
+    pub fn try_get(self) -> Result<RcuReadGuard<'a, P, false>, Self> {
+        if self.obj_ptr.is_null() {
+            return Err(self);
+        }
+        Ok(RcuReadGuard {
+            obj_ptr: self.obj_ptr,
+            // SAFETY: It is initialized. The layout is the same.
+            rcu: unsafe { core::mem::transmute::<&Rcu<P, true>, &Rcu<P, false>>(self.rcu) },
+            _inner_guard: self._inner_guard,
+        })
+    }
+}
+
+impl<P: OwnerPtr + Send, const NULLABLE: bool> RcuReadGuard<'_, P, NULLABLE> {
+    /// Tries to replace the already read pointer with a new pointer.
+    ///
+    /// If another thread has updated the pointer after the read, this
+    /// function will fail and return the new pointer. Otherwise, it will
+    /// replace the pointer with the new one and drop the old pointer after
+    /// the grace period.
+    ///
+    /// If spinning on this function, it is recommended to relax the CPU
+    /// or yield the task on failure. Otherwise contention will occur.
+    ///
+    /// This API does not help to avoid
+    /// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem).
+    pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> {
+        let new_ptr = <P as OwnerPtr>::into_raw(new_ptr).cast_mut();
+
+        if self
+            .rcu
+            .ptr
+            .compare_exchange(self.obj_ptr, new_ptr, AcqRel, Acquire)
+            .is_err()
+        {
+            // SAFETY: It was previously returned by `into_raw`.
+            return Err(unsafe { <P as OwnerPtr>::from_raw(new_ptr) });
+        }
+
+        if let Some(p) = NonNull::new(self.obj_ptr) {
+            // SAFETY: It was previously returned by `into_raw`.
+            unsafe { delay_drop::<P>(p) };
+        }
+
+        Ok(())
+    }
+}
+
+/// # Safety
+///
+/// The pointer must be previously returned by `into_raw` and the pointer
+/// must be only be dropped once.
+unsafe fn delay_drop<P: OwnerPtr + Send>(pointer: NonNull<<P as OwnerPtr>::Target>) {
+    // SAFETY: The pointer is not NULL.
+    let p = unsafe { <P as OwnerPtr>::from_raw(pointer.as_ptr().cast_const()) };
+    let rcu_monitor = RCU_MONITOR.get().unwrap();
+    rcu_monitor.after_grace_period(move || {
+        drop(p);
+    });
+}
+
 /// Finishes the current grace period.
 ///
 /// This function is called when the current grace period on current CPU is

ostd/src/sync/rcu/owner_ptr.rs

@@ -13,6 +13,9 @@ pub trait OwnerPtr: 'static {
     // TODO: allow ?Sized
     type Target;
 
+    /// Creates a new pointer with the given value.
+    fn new(value: Self::Target) -> Self;
+
     /// Converts to a raw pointer.
     ///
     /// If `Self` owns the object that it refers to (e.g., `Box<_>`), then
@@ -31,6 +34,10 @@ pub trait OwnerPtr: 'static {
 impl<T: 'static> OwnerPtr for Box<T> {
     type Target = T;
 
+    fn new(value: Self::Target) -> Self {
+        Box::new(value)
+    }
+
     fn into_raw(self) -> *const Self::Target {
         Box::into_raw(self) as *const _
     }
@@ -43,6 +50,10 @@ impl<T: 'static> OwnerPtr for Box<T> {
 impl<T: 'static> OwnerPtr for Arc<T> {
     type Target = T;
 
+    fn new(value: Self::Target) -> Self {
+        Arc::new(value)
+    }
+
     fn into_raw(self) -> *const Self::Target {
         Arc::into_raw(self)
     }
@@ -63,6 +74,10 @@ where
 {
     type Target = P::Target;
 
+    fn new(value: Self::Target) -> Self {
+        Some(P::new(value))
+    }
+
     fn into_raw(self) -> *const Self::Target {
         self.map(|p| <P as OwnerPtr>::into_raw(p))
             .unwrap_or(core::ptr::null())