Implement fallible memory operations on RISC-V platform

osdk/src/base_crate/riscv64.ld.template

@@ -30,6 +30,16 @@ SECTIONS
         PROVIDE(__etext = .);
     }
 
+    # The section to store exception table (ExTable).
+    # This table is used for recovering from specific exception handling faults
+    # occurring at known points in the code.
+    # Ref: /ostd/src/ex_table.rs
+    .ex_table               : AT(ADDR(.ex_table) - KERNEL_VMA_OFFSET) {
+        __ex_table = .;
+        KEEP(*(SORT(.ex_table)))
+        __ex_table_end = .;
+    }
+
     .rodata                 : AT(ADDR(.rodata) - KERNEL_VMA_OFFSET) {
         *(.rodata .rodata.*)
     }

ostd/src/arch/riscv/mm/atomic_cmpxchg_fallible.S

@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: MPL-2.0 */
+
+.text
+
+.option push
+# We explicitly add this directive due to a known bug in how rustc treats target
+# features in `global_asm`. Check https://github.com/rust-lang/rust/issues/111637#issuecomment-1870096878
+# for details.
+.option arch, rv64imac
+
+# Atomically compares and exchanges a 32-bit integer value. This function works
+# with exception handling and can recover from a page fault.
+#
+# Returns the previous value or `!0u64` if failed to update.
+.global __atomic_cmpxchg_fallible
+.type __atomic_cmpxchg_fallible, @function
+__atomic_cmpxchg_fallible: # (ptr: *mut u32, old_val: u32, new_val: u32) -> u64
+    li t2, {SSTATUS_SUM}
+    csrs sstatus, t2
+cmpxchg_load:
+    lr.w t0, (a0)
+    bne t0, a1, cmpxchg_done
+cmpxchg_store:
+    sc.w t1, a2, (a0)
+    bnez t1, cmpxchg_load
+cmpxchg_done:
+    mv a0, t0
+    csrc sstatus, t2
+    ret
+cmpxchg_fault:
+    li a0, -1
+    csrc sstatus, t2
+    ret
+.size __atomic_cmpxchg_fallible, .-__atomic_cmpxchg_fallible
+
+.option pop
+
+.pushsection .ex_table, "a", @progbits
+    .balign 8
+    .quad cmpxchg_load
+    .quad cmpxchg_fault
+    .quad cmpxchg_store
+    .quad cmpxchg_fault
+.popsection

ostd/src/arch/riscv/mm/atomic_load_fallible.S

@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: MPL-2.0 */
+
+.text
+
+# Atomically loads a 32-bit integer value. This function works with exception
+# handling and can recover from a page fault.
+#
+# Returns the loaded value or `!0u64` if failed to load.
+.global __atomic_load_fallible
+.type __atomic_load_fallible, @function
+__atomic_load_fallible: # (ptr: *const u32) -> u64;
+    li t0, {SSTATUS_SUM}
+    csrs sstatus, t0
+load:
+    lwu a0, (a0)
+    csrc sstatus, t0
+    ret
+load_fault:
+    li a0, -1
+    csrc sstatus, t0
+    ret
+.size __atomic_load_fallible, .-__atomic_load_fallible
+
+.pushsection .ex_table, "a", @progbits
+    .balign 8
+    .quad load
+    .quad load_fault
+.popsection

ostd/src/arch/riscv/mm/memcpy_fallible.S

@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: MPL-2.0 */
+
+.text
+
+# Copies `size` bytes from `src` to `dst`. This function works with exception
+# handling and can recover from a page fault. The source range must not overlap
+# with the destination range (In virtual address level. Their corresponding
+# physical addresses can be overlapped).
+#
+# Returns number of bytes that failed to copy.
+.global __memcpy_fallible
+.type __memcpy_fallible, @function
+__memcpy_fallible: # (dst: *mut u8, src: *const u8, size: usize) -> usize
+    li t1, {SSTATUS_SUM}
+    csrs sstatus, t1
+memcpy_loop:
+    beqz a2, memcpy_exit
+memcpy_load:
+    lb t0, (a1)
+memcpy_store:
+    sb t0, (a0)
+    addi a0, a0, 1
+    addi a1, a1, 1
+    addi a2, a2, -1
+    j memcpy_loop
+memcpy_exit:
+    mv a0, a2
+    csrc sstatus, t1
+    ret
+.size __memcpy_fallible, .-__memcpy_fallible
+
+.pushsection .ex_table, "a", @progbits
+    .balign 8
+    .quad memcpy_load
+    .quad memcpy_exit
+    .quad memcpy_store
+    .quad memcpy_exit
+.popsection

ostd/src/arch/riscv/mm/memset_fallible.S

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MPL-2.0 */
+
+.text
+
+# Sets `size` bytes of memory at `dst` to the byte value given by `value`.
+# This function works with exception handling and can recover from a page fault.
+#
+# Returns number of bytes that failed to set.
+.global __memset_fallible
+.type __memset_fallible, @function
+__memset_fallible: # (dst: *mut u8, value: u8, size: usize) -> usize
+    li t0, {SSTATUS_SUM}
+    csrs sstatus, t0
+memset_loop:
+    beqz a2, memset_exit
+memset_store:
+    sb a1, (a0)
+    addi a0, a0, 1
+    addi a2, a2, -1
+    j memset_loop
+memset_exit:
+    mv a0, a2
+    csrc sstatus, t0
+    ret
+.size __memset_fallible, .-__memset_fallible
+
+.pushsection .ex_table, "a", @progbits
+    .balign 8
+    .quad memset_store
+    .quad memset_exit
+.popsection

ostd/src/arch/riscv/mm/mod.rs

@@ -4,6 +4,9 @@ use alloc::fmt;
 use core::ops::Range;
 
 use spin::Once;
+pub(crate) use util::{
+    __atomic_cmpxchg_fallible, __atomic_load_fallible, __memcpy_fallible, __memset_fallible,
+};
 
 use crate::{
     arch::{
@@ -18,6 +21,8 @@ use crate::{
     Pod,
 };
 
+mod util;
+
 #[derive(Clone, Debug, Default)]
 pub(crate) struct PagingConsts {}
 
@@ -281,29 +286,3 @@ impl fmt::Debug for PageTableEntry {
             .finish()
     }
 }
-
-pub(crate) unsafe fn __memcpy_fallible(dst: *mut u8, src: *const u8, size: usize) -> usize {
-    // TODO: Implement this fallible operation.
-    unsafe { riscv::register::sstatus::set_sum() };
-    unsafe { core::ptr::copy(src, dst, size) };
-    0
-}
-
-pub(crate) unsafe fn __memset_fallible(dst: *mut u8, value: u8, size: usize) -> usize {
-    // TODO: Implement this fallible operation.
-    unsafe { riscv::register::sstatus::set_sum() };
-    unsafe { core::ptr::write_bytes(dst, value, size) };
-    0
-}
-
-pub(crate) unsafe fn __atomic_load_fallible(ptr: *const u32) -> u64 {
-    // TODO: Implement this fallible operation.
-    unsafe { riscv::register::sstatus::set_sum() };
-    unsafe { core::intrinsics::atomic_load_relaxed(ptr) as u64 }
-}
-
-pub(crate) unsafe fn __atomic_cmpxchg_fallible(ptr: *mut u32, old_val: u32, new_val: u32) -> u64 {
-    // TODO: Implement this fallible operation.
-    unsafe { riscv::register::sstatus::set_sum() };
-    unsafe { core::intrinsics::atomic_cxchg_relaxed_relaxed(ptr, old_val, new_val).0 as u64 }
-}

ostd/src/arch/riscv/mm/util.rs

@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: MPL-2.0
+
+use crate::arch::trap::SSTATUS_SUM;
+
+core::arch::global_asm!(include_str!("memcpy_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM);
+core::arch::global_asm!(include_str!("memset_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM);
+
+core::arch::global_asm!(include_str!("atomic_load_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM);
+core::arch::global_asm!(include_str!("atomic_cmpxchg_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM);
+
+extern "C" {
+    /// Copies `size` bytes from `src` to `dst`. This function works with exception handling
+    /// and can recover from page fault.
+    /// Returns number of bytes that failed to copy.
+    pub(crate) fn __memcpy_fallible(dst: *mut u8, src: *const u8, size: usize) -> usize;
+    /// Fills `size` bytes in the memory pointed to by `dst` with the value `value`.
+    /// This function works with exception handling and can recover from page fault.
+    /// Returns number of bytes that failed to set.
+    pub(crate) fn __memset_fallible(dst: *mut u8, value: u8, size: usize) -> usize;
+
+    /// Atomically loads a 32-bit integer value. This function works with exception handling
+    /// and can recover from page fault.
+    /// Returns the loaded value or `!0u64` if failed to load.
+    pub(crate) fn __atomic_load_fallible(ptr: *const u32) -> u64;
+    /// Atomically compares and exchanges a 32-bit integer value. This function works with
+    /// exception handling and can recover from page fault.
+    /// Returns the previous value or `!0u64` if failed to update.
+    pub(crate) fn __atomic_cmpxchg_fallible(ptr: *mut u32, old_val: u32, new_val: u32) -> u64;
+}

ostd/src/arch/riscv/trap/mod.rs

@@ -10,7 +10,7 @@ use core::sync::atomic::Ordering;
 use riscv::register::scause::{Interrupt, Trap};
 use spin::Once;
 pub use trap::TrapFrame;
-pub(super) use trap::{RawUserContext, SSTATUS_FS_MASK};
+pub(super) use trap::{RawUserContext, SSTATUS_FS_MASK, SSTATUS_SUM};
 
 use crate::{
     arch::{

ostd/src/arch/riscv/trap/trap.S

@@ -68,19 +68,20 @@ trap_from_user:
     STORE_SP x31, 31
 
     # save sp, sstatus, sepc
-    csrrw t0, sscratch, x0   # sscratch = 0 (kernel)
-    li t3, {SSTATUS_FS_MASK} # disable FPU to prevent unexpected usage of floating point in kernel space
+    csrrw t0, sscratch, x0                   # sscratch = 0 (kernel)
+    li t3, {SSTATUS_FS_MASK} | {SSTATUS_SUM} # disable FPU to prevent unexpected usage of floating point in kernel space
+                                             # disable U-mode memory access as it should be set on demand
     csrrc t1, sstatus, t3
     csrr t2, sepc
-    STORE_SP t0, 2           # save sp
-    STORE_SP t1, 32          # save sstatus
-    STORE_SP t2, 33          # save sepc
+    STORE_SP t0, 2                           # save sp
+    STORE_SP t1, 32                          # save sstatus
+    STORE_SP t2, 33                          # save sepc
 
-    andi t1, t1, 1 << 8      # sstatus.SPP == 1
+    andi t1, t1, 1 << 8                      # sstatus.SPP == 1
     beqz t1, end_trap_from_user
 end_trap_from_kernel:
-    mv a0, sp                # first arg is TrapFrame
-    lla ra, trap_return      # set return address
+    mv a0, sp                                # first arg is TrapFrame
+    lla ra, trap_return                      # set return address
 .extern trap_handler
     j trap_handler
 

ostd/src/arch/riscv/trap/trap.rs

@@ -49,8 +49,11 @@ global_asm!(
 /// FPU status bits.
 /// Reference: <https://riscv.github.io/riscv-isa-manual/snapshot/privileged/#sstatus>.
 pub(in crate::arch) const SSTATUS_FS_MASK: usize = 0b11 << 13;
+/// Supervisor User Memory access bit.
+/// Reference: <https://riscv.github.io/riscv-isa-manual/snapshot/privileged/#sstatus>.
+pub(in crate::arch) const SSTATUS_SUM: usize = 0b1 << 18;
 
-global_asm!(include_str!("trap.S"), SSTATUS_FS_MASK = const SSTATUS_FS_MASK);
+global_asm!(include_str!("trap.S"), SSTATUS_FS_MASK = const SSTATUS_FS_MASK, SSTATUS_SUM = const SSTATUS_SUM);
 
 /// Initialize interrupt handling for the current HART.
 ///