Implement a generic interface for memory barriers (full system/DMA/SMP). The interface uses a parameter to force user to specify their intent with barriers.
Provide `Read`, `Write`, `Full` orderings which map to the existing `rmb()`, `wmb()` and `mb()`. Generic is used here instead of providing individual standalone functions to reduce code duplication; for example, the `CONFIG_SMP` check in `smp_mb` is uniformly implemented for all SMP barriers. This could extend to `virt_mb`'s if they're introduced in the future. It would also make it easier if new ordering types are introduced in the future (e.g. `Acquire`, `Release`). Signed-off-by: Gary Guo <[email protected]> --- rust/kernel/sync/atomic/ordering.rs | 2 +- rust/kernel/sync/barrier.rs | 123 ++++++++++++++++++++++++++++-------- 2 files changed, 96 insertions(+), 29 deletions(-) diff --git a/rust/kernel/sync/atomic/ordering.rs b/rust/kernel/sync/atomic/ordering.rs index 3f103aa8db99..c4e732e7212f 100644 --- a/rust/kernel/sync/atomic/ordering.rs +++ b/rust/kernel/sync/atomic/ordering.rs @@ -15,7 +15,7 @@ //! - It provides ordering between the annotated operation and all the following memory accesses. //! - It provides ordering between all the preceding memory accesses and all the following memory //! accesses. -//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb()`). +//! - All the orderings are the same strength as a full memory barrier (i.e. `smp_mb(Full)`). //! - [`Relaxed`] provides no ordering except the dependency orderings. Dependency orderings are //! described in "DEPENDENCY RELATIONS" in [`LKMM`]'s [`explanation`]. //! diff --git a/rust/kernel/sync/barrier.rs b/rust/kernel/sync/barrier.rs index 8f2d435fcd94..54c527fdb760 100644 --- a/rust/kernel/sync/barrier.rs +++ b/rust/kernel/sync/barrier.rs @@ -7,6 +7,34 @@ //! //! [`LKMM`]: srctree/tools/memory-model/ +#![expect(private_bounds, reason = "sealed implementation")] + +/// Memory barrier orderings. +/// +/// The semantics of these orderings follows the [`LKMM`] definitions and rules. +/// +/// - [`Read`] provides ordering between preceding load operations and succeeding load operations. +/// - [`Write`] provides ordering between preceding store operations and succeeding store +/// operations. +/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory +/// accesses. +/// +/// [`LKMM`]: srctree/tools/memory-model/ +pub mod ordering { + pub use crate::sync::atomic::ordering::Full; + + /// The annotation type for read-read barrier ordering. + pub struct Read; + + /// The annotation type for write-write barrier ordering. + pub struct Write; +} + +pub use ordering::{Full, Read, Write}; + +struct Smp; +struct Dma; + /// A compiler barrier. /// /// A barrier that prevents compiler from reordering memory accesses across the barrier. @@ -19,43 +47,82 @@ pub(crate) fn barrier() { unsafe { core::arch::asm!("") }; } -/// A full memory barrier. +trait MemoryBarrier<Flavour = ()> { + fn run(); +} + +macro_rules! define_barrier { + ($([$flavour:ident])? $ordering:ident, $binding:ident) => { + impl MemoryBarrier$(<$flavour>)? for $ordering { + #[inline] + fn run() { + // SAFETY: barrier methods are safe to call. + unsafe { bindings::$binding() }; + } + } + }; +} + +define_barrier!(Full, mb); +define_barrier!(Read, rmb); +define_barrier!(Write, wmb); +define_barrier!([Dma] Full, dma_mb); +define_barrier!([Dma] Read, dma_rmb); +define_barrier!([Dma] Write, dma_wmb); +define_barrier!([Smp] Full, smp_mb); +define_barrier!([Smp] Read, smp_rmb); +define_barrier!([Smp] Write, smp_wmb); + +/// Memory barrier. /// /// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. -#[inline(always)] -pub fn smp_mb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_mb()` is safe to call. - unsafe { bindings::smp_mb() }; - } else { - barrier(); - } +/// +/// The specific forms of reordering can be specified using the parameter. +/// - `mb(Read)` provides a read-read barrier. +/// - `mb(Write)` provides a write-write barrier. +/// - `mb(Full)` provides a full barrier. +/// +/// # Examples +/// +/// ``` +/// # use kernel::sync::barrier::*; +/// mb(Read); +/// mb(Write); +/// mb(Full); +/// ``` +#[inline] +#[doc(alias = "rmb")] +#[doc(alias = "wmb")] +pub fn mb<T: MemoryBarrier>(_: T) { + T::run() } -/// A write-write memory barrier. +/// Memory barrier between CPUs. /// -/// A barrier that prevents compiler and CPU from reordering memory write accesses across the -/// barrier. -#[inline(always)] -pub fn smp_wmb() { +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other bus-mastering devices. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "smp_rmb")] +#[doc(alias = "smp_wmb")] +pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) { if cfg!(CONFIG_SMP) { - // SAFETY: `smp_wmb()` is safe to call. - unsafe { bindings::smp_wmb() }; + T::run() } else { - barrier(); + barrier() } } -/// A read-read memory barrier. +/// Memory barrier between local CPU and bus-mastering devices. /// -/// A barrier that prevents compiler and CPU from reordering memory read accesses across the -/// barrier. -#[inline(always)] -pub fn smp_rmb() { - if cfg!(CONFIG_SMP) { - // SAFETY: `smp_rmb()` is safe to call. - unsafe { bindings::smp_rmb() }; - } else { - barrier(); - } +/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier. +/// Does not prevent re-ordering with respect to other CPUs. +/// +/// See [`mb`] for usage. +#[inline] +#[doc(alias = "dma_rmb")] +#[doc(alias = "dma_wmb")] +pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) { + T::run() } -- 2.54.0
