diff --git a/osdk/src/commands/build/mod.rs b/osdk/src/commands/build/mod.rs index 65ebcd2ed..c8e76edae 100644 --- a/osdk/src/commands/build/mod.rs +++ b/osdk/src/commands/build/mod.rs @@ -224,6 +224,9 @@ fn build_kernel_elf( // It makes running on Intel CPUs after Ivy Bridge (2012) faster, but much slower // on older CPUs. rustflags.push("-C target-feature=+ermsb"); + } else if matches!(arch, Arch::RiscV64) { + // Enable the Zicbom extension for RISC-V to support cache block operations. + rustflags.push("-C target-feature=+zicbom"); } let mut command = cargo(); diff --git a/ostd/src/arch/loongarch/mm/mod.rs b/ostd/src/arch/loongarch/mm/mod.rs index d53e8a38e..8b2c41a0f 100644 --- a/ostd/src/arch/loongarch/mm/mod.rs +++ b/ostd/src/arch/loongarch/mm/mod.rs @@ -7,7 +7,7 @@ use crate::{ mm::{ page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, page_table::PageTableEntryTrait, - Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, + DmaDirection, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, }, Pod, }; @@ -102,6 +102,10 @@ pub(crate) fn tlb_flush_all_including_global() { } } +pub(crate) fn sync_dma_range(_range: Range, _direction: DmaDirection) { + todo!("Implement DMA synchronization for LoongArch64 architecture"); +} + /// Activates the given level 4 page table. /// /// "pgdl" or "pgdh" register doesn't have a field that encodes the cache policy, diff --git a/ostd/src/arch/riscv/mm/mod.rs b/ostd/src/arch/riscv/mm/mod.rs index 7fb5772e3..72cd6371d 100644 --- a/ostd/src/arch/riscv/mm/mod.rs +++ b/ostd/src/arch/riscv/mm/mod.rs @@ -3,12 +3,17 @@ use alloc::fmt; use core::ops::Range; +use spin::Once; + use crate::{ - arch::cpu::extension::{has_extensions, IsaExtensions}, + arch::{ + boot::DEVICE_TREE, + cpu::extension::{has_extensions, IsaExtensions}, + }, mm::{ page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, page_table::PageTableEntryTrait, - Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, + DmaDirection, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, }, Pod, }; @@ -83,6 +88,50 @@ pub(crate) fn tlb_flush_all_including_global() { riscv::asm::sfence_vma_all() } +pub(crate) fn sync_dma_range(range: Range, direction: DmaDirection) { + if has_extensions(IsaExtensions::ZICBOM) { + static CMO_MANAGEMENT_BLOCK_SIZE: Once = Once::new(); + CMO_MANAGEMENT_BLOCK_SIZE.call_once(|| { + DEVICE_TREE + .get() + .unwrap() + .cpus() + .find(|cpu| cpu.property("mmu-type").is_some()) + .expect("Failed to find an application CPU node in device tree") + .property("riscv,cbom-block-size") + .expect("Failed to find `riscv,cbom-block-size` property of the CPU node") + .as_usize() + .unwrap_or(64) + }); + + for addr in range.step_by(*CMO_MANAGEMENT_BLOCK_SIZE.get().unwrap()) { + // SAFETY: These are cache maintenance operations on a valid, owned + // memory range. They are required for correctness on systems with + // non-coherent DMA. + unsafe { + match direction { + DmaDirection::ToDevice => { + core::arch::asm!("cbo.clean ({})", in(reg) addr, options(nostack)) + } + DmaDirection::FromDevice => { + core::arch::asm!("cbo.inval ({})", in(reg) addr, options(nostack)); + } + DmaDirection::Bidirectional => { + core::arch::asm!("cbo.flush ({})", in(reg) addr, options(nostack)); + } + } + } + } + // Ensure that all cache operations have completed before proceeding. + // SAFETY: Safe because it is only a memory fence. + unsafe { + core::arch::asm!("fence rw, rw", options(nostack)); + } + } else { + // TODO: Implement DMA synchronization without ZICBOM support. + } +} + #[derive(Clone, Copy, Pod, Default)] #[repr(C)] pub(crate) struct PageTableEntry(usize); diff --git a/ostd/src/arch/x86/mm/mod.rs b/ostd/src/arch/x86/mm/mod.rs index 6ec97319b..ada363e09 100644 --- a/ostd/src/arch/x86/mm/mod.rs +++ b/ostd/src/arch/x86/mm/mod.rs @@ -13,7 +13,7 @@ use crate::{ mm::{ page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, page_table::PageTableEntryTrait, - Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, + DmaDirection, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, PAGE_SIZE, }, Pod, }; @@ -110,6 +110,12 @@ pub(crate) fn tlb_flush_all_including_global() { } } +pub(crate) fn sync_dma_range(_range: Range, _direction: DmaDirection) { + // The streaming DMA mapping in x86_64 is cache coherent, and does not + // require synchronization. + // Reference: , . +} + #[derive(Clone, Copy, Pod, Default)] #[repr(C)] pub(crate) struct PageTableEntry(usize); diff --git a/ostd/src/mm/dma/dma_stream.rs b/ostd/src/mm/dma/dma_stream.rs index 4d43c9fd1..031f52d0b 100644 --- a/ostd/src/mm/dma/dma_stream.rs +++ b/ostd/src/mm/dma/dma_stream.rs @@ -26,8 +26,6 @@ use crate::{ pub struct DmaStream { segment: USegment, start_daddr: Daddr, - /// TODO: remove this field when on x86. - #[expect(unused)] is_cache_coherent: bool, direction: DmaDirection, } @@ -123,28 +121,19 @@ impl DmaStream { /// /// [`read_bytes`]: crate::mm::VmIo::read_bytes /// [`write_bytes`]: crate::mm::VmIo::write_bytes - pub fn sync(&self, _byte_range: Range) -> Result<(), Error> { - cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")]{ - // The streaming DMA mapping in x86_64 is cache coherent, and does not require synchronization. - // Reference: , - Ok(()) - } else { - if _byte_range.end > self.size() { - return Err(Error::InvalidArgs); - } - if self.is_cache_coherent { - return Ok(()); - } - let _start_va = crate::mm::paddr_to_vaddr(self.segment.paddr()) as *const u8; - // TODO: Query the CPU for the cache line size via CPUID, we use 64 bytes as the cache line size here. - for _i in _byte_range.step_by(64) { - // TODO: Call the cache line flush command in the corresponding architecture. - todo!() - } - Ok(()) - } + pub fn sync(&self, byte_range: Range) -> Result<(), Error> { + if byte_range.end > self.size() { + return Err(Error::InvalidArgs); } + if self.is_cache_coherent { + return Ok(()); + } + + let start_vaddr = crate::mm::paddr_to_vaddr(self.segment.paddr()); + let range = (start_vaddr + byte_range.start)..(start_vaddr + byte_range.end); + crate::arch::mm::sync_dma_range(range, self.direction); + + Ok(()) } }