Samuka007/asterinas
1
0
Fork 0
mirror of https://github.com/asterinas/asterinas.git synced 2025-06-29 16:13:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a read-copy-update method to RCU and allow nullable RCU

Browse Source
This commit is contained in:
Zhang Junyang
2024-11-13 21:07:22 +08:00
committed by Tate, Hongliang Tian
parent 43985c737a
commit eee11fa813
3 changed files with 231 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ostd/src/sync/mod.rs
View File

@ -15,7 +15,7 @@ pub(crate) use self::{guard::GuardTransfer, rcu::finish_grace_period};
pub use self::{
guard::{LocalIrqDisabled, PreemptDisabled, WriteIrqDisabled},
mutex::{ArcMutexGuard, Mutex, MutexGuard},
rcu::{OwnerPtr, Rcu, RcuReadGuard},
rcu::{OwnerPtr, Rcu, RcuOption, RcuReadGuard},
rwarc::{RoArc, RwArc},
rwlock::{
ArcRwLockReadGuard, ArcRwLockUpgradeableGuard, ArcRwLockWriteGuard, RwLock,

286
ostd/src/sync/rcu/mod.rs
View File

@ -5,6 +5,7 @@
use core::{
marker::PhantomData,
ops::Deref,
ptr::NonNull,
sync::atomic::{
AtomicPtr,
Ordering::{AcqRel, Acquire},
@ -21,122 +22,263 @@ mod owner_ptr;
pub use owner_ptr::OwnerPtr;
/// Read-Copy Update Synchronization Mechanism
/// A Read-Copy Update (RCU) cell for sharing a pointer between threads.
///
/// The pointer should be a owning pointer with type `P`, which implements
/// [`OwnerPtr`]. For example, `P` can be `Box<T>` or `Arc<T>`.
///
/// # Overview
///
/// RCU avoids the use of lock primitives lock primitives while multiple threads
/// concurrently read and update elements that are linked through pointers and that
/// belong to shared data structures.
/// Read-Copy-Update (RCU) is a synchronization mechanism designed for high-
/// performance, low-latency read operations in concurrent systems. It allows
/// multiple readers to access shared data simultaneously without contention,
/// while writers can update the data safely in a way that does not disrupt
/// ongoing reads. RCU is particularly suited for situations where reads are
/// far more frequent than writes.
///
/// Whenever a thread is inserting or deleting elements of data structures in shared
/// memory, all readers are guaranteed to see and traverse either the older or the
/// new structure, therefore avoiding inconsistencies and allowing readers to not be
/// blocked by writers.
///
/// The type parameter `P` represents the data that this rcu is protecting. The type
/// parameter `P` must implement [`OwnerPtr`].
///
/// # Usage
///
/// It is used when performance of reads is crucial and is an example of space–time
/// tradeoff, enabling fast operations at the cost of more space.
///
/// Use [`Rcu`] in scenarios that require frequent reads and infrequent
/// updates (read-mostly).
///
/// Use [`Rcu`] in scenarios that require high real-time reading.
///
/// Rcu should not to be used in the scenarios that write-mostly and which need
/// consistent data.
/// The original design and implementation of RCU is described in paper _The
/// Read-Copy-Update Mechanism for Supporting Real-Time Applications on Shared-
/// Memory Multiprocessor Systems with Linux_ published on IBM Systems Journal
/// 47.2 (2008).
///
/// # Examples
///
/// ```
/// use aster_frame::sync::{Rcu, RcuReadGuard, RcuReclaimer};
/// use ostd::sync::Rcu;
///
/// let rcu = Rcu::new(Box::new(42));
///
/// // Read the data protected by rcu
/// let rcu_guard = rcu.read();
///
/// assert_eq!(*rcu_guard, Some(&42));
///
/// rcu_guard.compare_exchange(Box::new(43)).unwrap();
///
/// let rcu_guard = rcu.read();
///
/// assert_eq!(*rcu_guard, Some(&43));
/// ```
#[repr(transparent)]
pub struct Rcu<P: OwnerPtr, const NULLABLE: bool = false> {
ptr: AtomicPtr<<P as OwnerPtr>::Target>,
// We want to implement Send and Sync explicitly.
// Having a pointer field prevents them from being implemented
// automatically by the compiler.
_marker: PhantomData<*const P::Target>,
}
/// A Read-Copy Update (RCU) cell for sharing a _nullable_ pointer.
///
/// This is a variant of [`Rcu`] that allows the contained pointer to be null.
/// So that it can implement `Rcu<Option<P>>` where `P` is not a nullable
/// pointer. It is the same as [`Rcu`] in other aspects.
///
/// # Examples
///
/// ```
/// use ostd::sync::RcuOption;
///
/// // Also allows lazy initialization.
/// static RCU: RcuOption<Box<usize>> = RcuOption::new_none();
///
/// // Not initialized yet.
/// assert!(RCU.read().try_get().is_none());
///
/// // Initialize the data protected by RCU.
/// RCU.update(Box::new(42));
///
/// // Read the data protected by RCU
/// {
/// let rcu_guard = rcu.get();
/// let rcu_guard = RCU.read().try_get().unwrap();
/// assert_eq!(*rcu_guard, 42);
/// }
///
/// // Update the data protected by rcu
/// // Update the data protected by RCU
/// {
/// let reclaimer = rcu.replace(Box::new(43));
/// let rcu_guard = RCU.read().try_get().unwrap();
///
/// let rcu_guard = rcu.get();
/// rcu_guard.compare_exchange(Box::new(43)).unwrap();
///
/// let rcu_guard = RCU.read().try_get().unwrap();
/// assert_eq!(*rcu_guard, 43);
/// }
/// ```
pub struct Rcu<P: OwnerPtr> {
ptr: AtomicPtr<<P as OwnerPtr>::Target>,
marker: PhantomData<P::Target>,
pub type RcuOption<P> = Rcu<P, true>;
// SAFETY: It is apparent that if `P::Target` is `Send`, then `Rcu<P>` is `Send`.
unsafe impl<P: OwnerPtr, const NULLABLE: bool> Send for Rcu<P, NULLABLE> where
<P as OwnerPtr>::Target: Send
{
}
impl<P: OwnerPtr> Rcu<P> {
/// Creates a new instance of Rcu with the given pointer.
pub fn new(ptr: P) -> Self {
let ptr = AtomicPtr::new(OwnerPtr::into_raw(ptr) as *mut _);
// SAFETY: To implement `Sync` for `Rcu<P>`, we need to meet two conditions:
// 1. `P::Target` must be `Sync` because `Rcu::get` allows concurrent access.
// 2. `P::Target` must be `Send` because `Rcu::update` may obtain an object
// of `P` created on another thread.
unsafe impl<P: OwnerPtr, const NULLABLE: bool> Sync for Rcu<P, NULLABLE> where
<P as OwnerPtr>::Target: Send + Sync
{
}
// Non-nullable RCU cell.
impl<P: OwnerPtr> Rcu<P, false> {
/// Creates a new RCU cell with the given pointer.
pub fn new(pointer: P) -> Self {
let ptr = <P as OwnerPtr>::into_raw(pointer).cast_mut();
let ptr = AtomicPtr::new(ptr);
Self {
ptr,
marker: PhantomData,
_marker: PhantomData,
}
}
}
// Nullable RCU cell.
impl<P: OwnerPtr> Rcu<P, true> {
/// Creates a new uninitialized RCU cell.
///
/// Initialization can be done by calling
/// [`RcuReadGuard::compare_exchange`] after getting a read
/// guard using [`Rcu::read`]. Then only the first initialization will be
/// successful. If initialization can be done multiple times, using
/// [`Rcu::update`] is fine.
pub const fn new_none() -> Self {
let ptr = AtomicPtr::new(core::ptr::null_mut());
Self {
ptr,
_marker: PhantomData,
}
}
}
impl<P: OwnerPtr + Send, const NULLABLE: bool> Rcu<P, NULLABLE> {
/// Replaces the current pointer with a new pointer.
///
/// This function updates the pointer to the new pointer regardless of the
/// original pointer. If the original pointer is not NULL, it will be
/// dropped after the grace period.
///
/// Oftentimes this function is not recommended unless you have
/// synchronized writes with locks. Otherwise, you can use [`Self::read`]
/// and then [`RcuReadGuard::compare_exchange`] to update the pointer.
pub fn update(&self, new_ptr: P) {
let new_ptr = <P as OwnerPtr>::into_raw(new_ptr).cast_mut();
let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel);
if let Some(p) = NonNull::new(old_raw_ptr) {
// SAFETY: It was previously returned by `into_raw`.
unsafe { delay_drop::<P>(p) };
}
}
/// Retrieves a read guard for the RCU mechanism.
/// Retrieves a read guard for the RCU cell.
///
/// This method returns a `RcuReadGuard` which allows read-only access to the
/// underlying data protected by the RCU mechanism.
pub fn get(&self) -> RcuReadGuard<'_, P> {
/// The guard allows read-only access to the data protected by RCU.
///
/// If the RCU cell is nullable, the guard will be nullable and you can
/// only dereference it after checking with [`RcuReadGuard::try_get`].
/// If the RCU cell is non-nullable, the guard will be non-nullable and
/// you can dereference it directly.
pub fn read(&self) -> RcuReadGuard<'_, P, NULLABLE> {
let guard = disable_preempt();
let obj = unsafe { &*self.ptr.load(Acquire) };
RcuReadGuard {
obj,
_rcu: self,
obj_ptr: self.ptr.load(Acquire),
rcu: self,
_inner_guard: guard,
}
}
}
impl<P: OwnerPtr + Send> Rcu<P> {
/// Replaces the current pointer with a new pointer.
pub fn replace(&self, new_ptr: P) {
let new_ptr = <P as OwnerPtr>::into_raw(new_ptr) as *mut _;
let old_ptr = {
let old_raw_ptr = self.ptr.swap(new_ptr, AcqRel);
// SAFETY: It is valid because it was previously returned by `into_raw`.
unsafe { <P as OwnerPtr>::from_raw(old_raw_ptr) }
};
let rcu_monitor = RCU_MONITOR.get().unwrap();
rcu_monitor.after_grace_period(move || {
drop(old_ptr);
});
}
}
/// A guard that allows read-only access to the data protected by the RCU
/// mechanism.
///
/// Note that the data read can be outdated if the data is updated by another
/// task after acquiring the guard.
pub struct RcuReadGuard<'a, P: OwnerPtr> {
obj: &'a <P as OwnerPtr>::Target,
_rcu: &'a Rcu<P>,
/// A guard that allows read-only access to the initialized data protected
/// by the RCU mechanism.
pub struct RcuReadGuard<'a, P: OwnerPtr, const NULLABLE: bool> {
/// If maybe uninitialized, the pointer can be NULL.
obj_ptr: *mut <P as OwnerPtr>::Target,
rcu: &'a Rcu<P, NULLABLE>,
_inner_guard: DisabledPreemptGuard,
}
impl<P: OwnerPtr> Deref for RcuReadGuard<'_, P> {
// Non-nullable RCU guard can be directly dereferenced.
impl<P: OwnerPtr> Deref for RcuReadGuard<'_, P, false> {
type Target = <P as OwnerPtr>::Target;
fn deref(&self) -> &Self::Target {
self.obj
// SAFETY: Since the preemption is disabled, the pointer is valid
// because other writers won't release the allocation until this task
// passes the quiescent state.
// And this pointer is not NULL.
unsafe { &*self.obj_ptr }
}
}
// Nullable RCU guard can be dereferenced after checking.
impl<'a, P: OwnerPtr> RcuReadGuard<'a, P, true> {
/// Tries to get the initialized read guard.
///
/// If the RCU cell is not initialized, this function will return
/// [`Err`] with the guard itself unchanged. Otherwise a dereferenceable
/// read guard will be returned.
pub fn try_get(self) -> Result<RcuReadGuard<'a, P, false>, Self> {
if self.obj_ptr.is_null() {
return Err(self);
}
Ok(RcuReadGuard {
obj_ptr: self.obj_ptr,
// SAFETY: It is initialized. The layout is the same.
rcu: unsafe { core::mem::transmute::<&Rcu<P, true>, &Rcu<P, false>>(self.rcu) },
_inner_guard: self._inner_guard,
})
}
}
impl<P: OwnerPtr + Send, const NULLABLE: bool> RcuReadGuard<'_, P, NULLABLE> {
/// Tries to replace the already read pointer with a new pointer.
///
/// If another thread has updated the pointer after the read, this
/// function will fail and return the new pointer. Otherwise, it will
/// replace the pointer with the new one and drop the old pointer after
/// the grace period.
///
/// If spinning on this function, it is recommended to relax the CPU
/// or yield the task on failure. Otherwise contention will occur.
///
/// This API does not help to avoid
/// [the ABA problem](https://en.wikipedia.org/wiki/ABA_problem).
pub fn compare_exchange(self, new_ptr: P) -> Result<(), P> {
let new_ptr = <P as OwnerPtr>::into_raw(new_ptr).cast_mut();
if self
.rcu
.ptr
.compare_exchange(self.obj_ptr, new_ptr, AcqRel, Acquire)
.is_err()
{
// SAFETY: It was previously returned by `into_raw`.
return Err(unsafe { <P as OwnerPtr>::from_raw(new_ptr) });
}
if let Some(p) = NonNull::new(self.obj_ptr) {
// SAFETY: It was previously returned by `into_raw`.
unsafe { delay_drop::<P>(p) };
}
Ok(())
}
}
/// # Safety
///
/// The pointer must be previously returned by `into_raw` and the pointer
/// must be only be dropped once.
unsafe fn delay_drop<P: OwnerPtr + Send>(pointer: NonNull<<P as OwnerPtr>::Target>) {
// SAFETY: The pointer is not NULL.
let p = unsafe { <P as OwnerPtr>::from_raw(pointer.as_ptr().cast_const()) };
let rcu_monitor = RCU_MONITOR.get().unwrap();
rcu_monitor.after_grace_period(move || {
drop(p);
});
}
/// Finishes the current grace period.
///
/// This function is called when the current grace period on current CPU is

15
ostd/src/sync/rcu/owner_ptr.rs
View File

@ -13,6 +13,9 @@ pub trait OwnerPtr: 'static {
// TODO: allow ?Sized
type Target;
/// Creates a new pointer with the given value.
fn new(value: Self::Target) -> Self;
/// Converts to a raw pointer.
///
/// If `Self` owns the object that it refers to (e.g., `Box<_>`), then
@ -31,6 +34,10 @@ pub trait OwnerPtr: 'static {
impl<T: 'static> OwnerPtr for Box<T> {
type Target = T;
fn new(value: Self::Target) -> Self {
Box::new(value)
}
fn into_raw(self) -> *const Self::Target {
Box::into_raw(self) as *const _
}
@ -43,6 +50,10 @@ impl<T: 'static> OwnerPtr for Box<T> {
impl<T: 'static> OwnerPtr for Arc<T> {
type Target = T;
fn new(value: Self::Target) -> Self {
Arc::new(value)
}
fn into_raw(self) -> *const Self::Target {
Arc::into_raw(self)
}
@ -63,6 +74,10 @@ where
{
type Target = P::Target;
fn new(value: Self::Target) -> Self {
Some(P::new(value))
}
fn into_raw(self) -> *const Self::Target {
self.map(|p| <P as OwnerPtr>::into_raw(p))
.unwrap_or(core::ptr::null())