diff --git a/osdk/src/base_crate/riscv64.ld.template b/osdk/src/base_crate/riscv64.ld.template index ec008db46..19b69eba4 100644 --- a/osdk/src/base_crate/riscv64.ld.template +++ b/osdk/src/base_crate/riscv64.ld.template @@ -30,6 +30,16 @@ SECTIONS PROVIDE(__etext = .); } + # The section to store exception table (ExTable). + # This table is used for recovering from specific exception handling faults + # occurring at known points in the code. + # Ref: /ostd/src/ex_table.rs + .ex_table : AT(ADDR(.ex_table) - KERNEL_VMA_OFFSET) { + __ex_table = .; + KEEP(*(SORT(.ex_table))) + __ex_table_end = .; + } + .rodata : AT(ADDR(.rodata) - KERNEL_VMA_OFFSET) { *(.rodata .rodata.*) } diff --git a/ostd/src/arch/riscv/mm/atomic_cmpxchg_fallible.S b/ostd/src/arch/riscv/mm/atomic_cmpxchg_fallible.S new file mode 100644 index 000000000..fafe6dd6e --- /dev/null +++ b/ostd/src/arch/riscv/mm/atomic_cmpxchg_fallible.S @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MPL-2.0 */ + +.text + +.option push +# We explicitly add this directive due to a known bug in how rustc treats target +# features in `global_asm`. Check https://github.com/rust-lang/rust/issues/111637#issuecomment-1870096878 +# for details. +.option arch, rv64imac + +# Atomically compares and exchanges a 32-bit integer value. This function works +# with exception handling and can recover from a page fault. +# +# Returns the previous value or `!0u64` if failed to update. +.global __atomic_cmpxchg_fallible +.type __atomic_cmpxchg_fallible, @function +__atomic_cmpxchg_fallible: # (ptr: *mut u32, old_val: u32, new_val: u32) -> u64 + li t2, {SSTATUS_SUM} + csrs sstatus, t2 +cmpxchg_load: + lr.w t0, (a0) + bne t0, a1, cmpxchg_done +cmpxchg_store: + sc.w t1, a2, (a0) + bnez t1, cmpxchg_load +cmpxchg_done: + mv a0, t0 + csrc sstatus, t2 + ret +cmpxchg_fault: + li a0, -1 + csrc sstatus, t2 + ret +.size __atomic_cmpxchg_fallible, .-__atomic_cmpxchg_fallible + +.option pop + +.pushsection .ex_table, "a", @progbits + .balign 8 + .quad cmpxchg_load + .quad cmpxchg_fault + .quad cmpxchg_store + .quad cmpxchg_fault +.popsection diff --git a/ostd/src/arch/riscv/mm/atomic_load_fallible.S b/ostd/src/arch/riscv/mm/atomic_load_fallible.S new file mode 100644 index 000000000..c27eb9daf --- /dev/null +++ b/ostd/src/arch/riscv/mm/atomic_load_fallible.S @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MPL-2.0 */ + +.text + +# Atomically loads a 32-bit integer value. This function works with exception +# handling and can recover from a page fault. +# +# Returns the loaded value or `!0u64` if failed to load. +.global __atomic_load_fallible +.type __atomic_load_fallible, @function +__atomic_load_fallible: # (ptr: *const u32) -> u64; + li t0, {SSTATUS_SUM} + csrs sstatus, t0 +load: + lwu a0, (a0) + csrc sstatus, t0 + ret +load_fault: + li a0, -1 + csrc sstatus, t0 + ret +.size __atomic_load_fallible, .-__atomic_load_fallible + +.pushsection .ex_table, "a", @progbits + .balign 8 + .quad load + .quad load_fault +.popsection diff --git a/ostd/src/arch/riscv/mm/memcpy_fallible.S b/ostd/src/arch/riscv/mm/memcpy_fallible.S new file mode 100644 index 000000000..0e2bf0ef2 --- /dev/null +++ b/ostd/src/arch/riscv/mm/memcpy_fallible.S @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: MPL-2.0 */ + +.text + +# Copies `size` bytes from `src` to `dst`. This function works with exception +# handling and can recover from a page fault. The source range must not overlap +# with the destination range (In virtual address level. Their corresponding +# physical addresses can be overlapped). +# +# Returns number of bytes that failed to copy. +.global __memcpy_fallible +.type __memcpy_fallible, @function +__memcpy_fallible: # (dst: *mut u8, src: *const u8, size: usize) -> usize + li t1, {SSTATUS_SUM} + csrs sstatus, t1 +memcpy_loop: + beqz a2, memcpy_exit +memcpy_load: + lb t0, (a1) +memcpy_store: + sb t0, (a0) + addi a0, a0, 1 + addi a1, a1, 1 + addi a2, a2, -1 + j memcpy_loop +memcpy_exit: + mv a0, a2 + csrc sstatus, t1 + ret +.size __memcpy_fallible, .-__memcpy_fallible + +.pushsection .ex_table, "a", @progbits + .balign 8 + .quad memcpy_load + .quad memcpy_exit + .quad memcpy_store + .quad memcpy_exit +.popsection diff --git a/ostd/src/arch/riscv/mm/memset_fallible.S b/ostd/src/arch/riscv/mm/memset_fallible.S new file mode 100644 index 000000000..08138420e --- /dev/null +++ b/ostd/src/arch/riscv/mm/memset_fallible.S @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MPL-2.0 */ + +.text + +# Sets `size` bytes of memory at `dst` to the byte value given by `value`. +# This function works with exception handling and can recover from a page fault. +# +# Returns number of bytes that failed to set. +.global __memset_fallible +.type __memset_fallible, @function +__memset_fallible: # (dst: *mut u8, value: u8, size: usize) -> usize + li t0, {SSTATUS_SUM} + csrs sstatus, t0 +memset_loop: + beqz a2, memset_exit +memset_store: + sb a1, (a0) + addi a0, a0, 1 + addi a2, a2, -1 + j memset_loop +memset_exit: + mv a0, a2 + csrc sstatus, t0 + ret +.size __memset_fallible, .-__memset_fallible + +.pushsection .ex_table, "a", @progbits + .balign 8 + .quad memset_store + .quad memset_exit +.popsection diff --git a/ostd/src/arch/riscv/mm/mod.rs b/ostd/src/arch/riscv/mm/mod.rs index 72cd6371d..2d77b73fc 100644 --- a/ostd/src/arch/riscv/mm/mod.rs +++ b/ostd/src/arch/riscv/mm/mod.rs @@ -4,6 +4,9 @@ use alloc::fmt; use core::ops::Range; use spin::Once; +pub(crate) use util::{ + __atomic_cmpxchg_fallible, __atomic_load_fallible, __memcpy_fallible, __memset_fallible, +}; use crate::{ arch::{ @@ -18,6 +21,8 @@ use crate::{ Pod, }; +mod util; + #[derive(Clone, Debug, Default)] pub(crate) struct PagingConsts {} @@ -281,29 +286,3 @@ impl fmt::Debug for PageTableEntry { .finish() } } - -pub(crate) unsafe fn __memcpy_fallible(dst: *mut u8, src: *const u8, size: usize) -> usize { - // TODO: Implement this fallible operation. - unsafe { riscv::register::sstatus::set_sum() }; - unsafe { core::ptr::copy(src, dst, size) }; - 0 -} - -pub(crate) unsafe fn __memset_fallible(dst: *mut u8, value: u8, size: usize) -> usize { - // TODO: Implement this fallible operation. - unsafe { riscv::register::sstatus::set_sum() }; - unsafe { core::ptr::write_bytes(dst, value, size) }; - 0 -} - -pub(crate) unsafe fn __atomic_load_fallible(ptr: *const u32) -> u64 { - // TODO: Implement this fallible operation. - unsafe { riscv::register::sstatus::set_sum() }; - unsafe { core::intrinsics::atomic_load_relaxed(ptr) as u64 } -} - -pub(crate) unsafe fn __atomic_cmpxchg_fallible(ptr: *mut u32, old_val: u32, new_val: u32) -> u64 { - // TODO: Implement this fallible operation. - unsafe { riscv::register::sstatus::set_sum() }; - unsafe { core::intrinsics::atomic_cxchg_relaxed_relaxed(ptr, old_val, new_val).0 as u64 } -} diff --git a/ostd/src/arch/riscv/mm/util.rs b/ostd/src/arch/riscv/mm/util.rs new file mode 100644 index 000000000..e86a18ec0 --- /dev/null +++ b/ostd/src/arch/riscv/mm/util.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MPL-2.0 + +use crate::arch::trap::SSTATUS_SUM; + +core::arch::global_asm!(include_str!("memcpy_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM); +core::arch::global_asm!(include_str!("memset_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM); + +core::arch::global_asm!(include_str!("atomic_load_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM); +core::arch::global_asm!(include_str!("atomic_cmpxchg_fallible.S"), SSTATUS_SUM = const SSTATUS_SUM); + +extern "C" { + /// Copies `size` bytes from `src` to `dst`. This function works with exception handling + /// and can recover from page fault. + /// Returns number of bytes that failed to copy. + pub(crate) fn __memcpy_fallible(dst: *mut u8, src: *const u8, size: usize) -> usize; + /// Fills `size` bytes in the memory pointed to by `dst` with the value `value`. + /// This function works with exception handling and can recover from page fault. + /// Returns number of bytes that failed to set. + pub(crate) fn __memset_fallible(dst: *mut u8, value: u8, size: usize) -> usize; + + /// Atomically loads a 32-bit integer value. This function works with exception handling + /// and can recover from page fault. + /// Returns the loaded value or `!0u64` if failed to load. + pub(crate) fn __atomic_load_fallible(ptr: *const u32) -> u64; + /// Atomically compares and exchanges a 32-bit integer value. This function works with + /// exception handling and can recover from page fault. + /// Returns the previous value or `!0u64` if failed to update. + pub(crate) fn __atomic_cmpxchg_fallible(ptr: *mut u32, old_val: u32, new_val: u32) -> u64; +} diff --git a/ostd/src/arch/riscv/trap/mod.rs b/ostd/src/arch/riscv/trap/mod.rs index 94734c99e..f7165f7ee 100644 --- a/ostd/src/arch/riscv/trap/mod.rs +++ b/ostd/src/arch/riscv/trap/mod.rs @@ -10,7 +10,7 @@ use core::sync::atomic::Ordering; use riscv::register::scause::{Interrupt, Trap}; use spin::Once; pub use trap::TrapFrame; -pub(super) use trap::{RawUserContext, SSTATUS_FS_MASK}; +pub(super) use trap::{RawUserContext, SSTATUS_FS_MASK, SSTATUS_SUM}; use crate::{ arch::{ diff --git a/ostd/src/arch/riscv/trap/trap.S b/ostd/src/arch/riscv/trap/trap.S index 110d3c426..377feafbd 100644 --- a/ostd/src/arch/riscv/trap/trap.S +++ b/ostd/src/arch/riscv/trap/trap.S @@ -68,19 +68,20 @@ trap_from_user: STORE_SP x31, 31 # save sp, sstatus, sepc - csrrw t0, sscratch, x0 # sscratch = 0 (kernel) - li t3, {SSTATUS_FS_MASK} # disable FPU to prevent unexpected usage of floating point in kernel space + csrrw t0, sscratch, x0 # sscratch = 0 (kernel) + li t3, {SSTATUS_FS_MASK} | {SSTATUS_SUM} # disable FPU to prevent unexpected usage of floating point in kernel space + # disable U-mode memory access as it should be set on demand csrrc t1, sstatus, t3 csrr t2, sepc - STORE_SP t0, 2 # save sp - STORE_SP t1, 32 # save sstatus - STORE_SP t2, 33 # save sepc + STORE_SP t0, 2 # save sp + STORE_SP t1, 32 # save sstatus + STORE_SP t2, 33 # save sepc - andi t1, t1, 1 << 8 # sstatus.SPP == 1 + andi t1, t1, 1 << 8 # sstatus.SPP == 1 beqz t1, end_trap_from_user end_trap_from_kernel: - mv a0, sp # first arg is TrapFrame - lla ra, trap_return # set return address + mv a0, sp # first arg is TrapFrame + lla ra, trap_return # set return address .extern trap_handler j trap_handler diff --git a/ostd/src/arch/riscv/trap/trap.rs b/ostd/src/arch/riscv/trap/trap.rs index cb327d147..29ca8967d 100644 --- a/ostd/src/arch/riscv/trap/trap.rs +++ b/ostd/src/arch/riscv/trap/trap.rs @@ -49,8 +49,11 @@ global_asm!( /// FPU status bits. /// Reference: . pub(in crate::arch) const SSTATUS_FS_MASK: usize = 0b11 << 13; +/// Supervisor User Memory access bit. +/// Reference: . +pub(in crate::arch) const SSTATUS_SUM: usize = 0b1 << 18; -global_asm!(include_str!("trap.S"), SSTATUS_FS_MASK = const SSTATUS_FS_MASK); +global_asm!(include_str!("trap.S"), SSTATUS_FS_MASK = const SSTATUS_FS_MASK, SSTATUS_SUM = const SSTATUS_SUM); /// Initialize interrupt handling for the current HART. ///