diff --git a/ostd/src/arch/loongarch/mm/mod.rs b/ostd/src/arch/loongarch/mm/mod.rs index 06e44030c..e6fcd1ece 100644 --- a/ostd/src/arch/loongarch/mm/mod.rs +++ b/ostd/src/arch/loongarch/mm/mod.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -use alloc::fmt; use core::{arch::asm, intrinsics::AtomicOrdering::Relaxed, ops::Range}; use crate::{ @@ -8,8 +7,10 @@ use crate::{ mm::{ PAGE_SIZE, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, dma::DmaDirection, - page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, - page_table::PageTableEntryTrait, + page_prop::{ + CachePolicy, PageFlags, PageProperty, PageTableFlags, PrivilegedPageFlags as PrivFlags, + }, + page_table::{PteScalar, PteTrait}, }, }; @@ -30,7 +31,7 @@ bitflags::bitflags! { #[derive(Pod)] #[repr(C)] /// Possible flags for a page table entry. - pub(crate) struct PageTableFlags: usize { + pub(crate) struct PteFlags: usize { /// Specifies whether the mapped frame is valid. const VALID = 1 << 0; /// Whether the memory area represented by this entry is modified. @@ -119,10 +120,6 @@ pub(crate) unsafe fn sync_dma_range(_range: Range) { unreachable!("`can_sync_dma()` never returns `true`"); } -#[derive(Clone, Copy, Pod, Default)] -#[repr(C)] -pub(crate) struct PageTableEntry(usize); - /// Activates the given root-level page table. /// /// "pgdl" or "pgdh" register doesn't have a field that encodes the cache policy, @@ -148,64 +145,47 @@ pub(crate) fn current_page_table_paddr() -> Paddr { pgdl } -impl PageTableEntry { - const PHYS_ADDR_MASK: usize = 0x0000_FFFF_FFFF_F000; +#[derive(Debug, Clone, Copy, Pod, Default)] +#[repr(C)] +pub(crate) struct PageTableEntry(usize); - fn is_user(&self) -> bool { - self.0 & PageTableFlags::PLVL.bits() != 0 && self.0 & PageTableFlags::PLVH.bits() != 0 - } - - fn is_huge(&self) -> bool { - if self.0 & PageTableFlags::IS_BASIC.bits() != 0 { - false - } else { - self.0 & PageTableFlags::GLOBAL_OR_HUGE.bits() != 0 - } - } - - fn is_global(&self) -> bool { - if self.0 & PageTableFlags::IS_BASIC.bits() != 0 { - self.0 & PageTableFlags::GLOBAL_OR_HUGE.bits() != 0 - } else { - self.0 & PageTableFlags::GLOBAL_IN_HUGE.bits() != 0 - } - } -} - -/// Parse a bit-flag bits `val` in the representation of `from` to `to` in bits. +/// Parses a bit-flag bits `val` in the representation of `from` to `to` in bits. macro_rules! parse_flags { ($val:expr, $from:expr, $to:expr) => { (($val as usize & $from.bits() as usize) >> $from.bits().ilog2() << $to.bits().ilog2()) }; } -impl PodOnce for PageTableEntry {} +impl PageTableEntry { + const PHYS_ADDR_MASK: usize = 0x0000_FFFF_FFFF_F000; -impl PageTableEntryTrait for PageTableEntry { - fn is_present(&self) -> bool { - self.0 & PageTableFlags::VALID.bits() != 0 + fn is_user(&self) -> bool { + self.0 & PteFlags::PLVL.bits() != 0 && self.0 & PteFlags::PLVH.bits() != 0 } - fn new_page(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self { - let flags = if level == 1 { - PageTableFlags::IS_BASIC.bits() + fn is_huge(&self) -> bool { + if self.0 & PteFlags::IS_BASIC.bits() != 0 { + false } else { - PageTableFlags::GLOBAL_OR_HUGE.bits() - }; - let mut pte = Self(paddr & Self::PHYS_ADDR_MASK); - pte.set_prop(prop); - let pte = pte.0 | flags; - Self(pte) + self.0 & PteFlags::GLOBAL_OR_HUGE.bits() != 0 + } } - fn new_pt(paddr: Paddr) -> Self { - Self(paddr & Self::PHYS_ADDR_MASK | PageTableFlags::VALID.bits()) + fn is_global(&self) -> bool { + if self.0 & PteFlags::IS_BASIC.bits() != 0 { + self.0 & PteFlags::GLOBAL_OR_HUGE.bits() != 0 + } else { + self.0 & PteFlags::GLOBAL_IN_HUGE.bits() != 0 + } + } + + fn is_last(&self, level: PagingLevel) -> bool { + level == 1 || self.is_huge() } fn paddr(&self) -> Paddr { if self.is_huge() { - let paddr = - (self.0 & Self::PHYS_ADDR_MASK & !PageTableFlags::GLOBAL_IN_HUGE.bits()) >> 12; + let paddr = (self.0 & Self::PHYS_ADDR_MASK & !PteFlags::GLOBAL_IN_HUGE.bits()) >> 12; paddr << 12 } else { let ppn = (self.0 & Self::PHYS_ADDR_MASK) >> 12; @@ -214,14 +194,15 @@ impl PageTableEntryTrait for PageTableEntry { } fn prop(&self) -> PageProperty { - let flags = (parse_flags!(!(self.0), PageTableFlags::NOT_READABLE, PageFlags::R)) - | (parse_flags!(self.0, PageTableFlags::WRITABLE, PageFlags::W)) - | (parse_flags!(!(self.0), PageTableFlags::NOT_EXECUTABLE, PageFlags::X)) + let flags = parse_flags!(!(self.0), PteFlags::NOT_READABLE, PageFlags::R) + | parse_flags!(self.0, PteFlags::WRITABLE, PageFlags::W) + | parse_flags!(!(self.0), PteFlags::NOT_EXECUTABLE, PageFlags::X) // TODO: How to get the accessed bit in loongarch? - | (parse_flags!(self.0, PageTableFlags::PRESENT, PageFlags::ACCESSED)) - | (parse_flags!(self.0, PageTableFlags::DIRTY, PageFlags::DIRTY)) - | (parse_flags!(self.0, PageTableFlags::RSV2, PageFlags::AVAIL2)); - let mut priv_flags = parse_flags!(self.0, PageTableFlags::RSV1, PrivFlags::AVAIL1); + | parse_flags!(self.0, PteFlags::PRESENT, PageFlags::ACCESSED) + | parse_flags!(self.0, PteFlags::DIRTY, PageFlags::DIRTY) + | parse_flags!(self.0, PteFlags::RSV2, PageFlags::AVAIL2); + + let mut priv_flags = parse_flags!(self.0, PteFlags::RSV1, PrivFlags::AVAIL1); if self.is_user() { priv_flags |= PrivFlags::USER.bits() as usize; } @@ -229,9 +210,9 @@ impl PageTableEntryTrait for PageTableEntry { priv_flags |= PrivFlags::GLOBAL.bits() as usize; } - let cache = if self.0 & PageTableFlags::MATL.bits() != 0 { + let cache = if self.0 & PteFlags::MATL.bits() != 0 { CachePolicy::Writeback - } else if self.0 & PageTableFlags::MATH.bits() != 0 { + } else if self.0 & PteFlags::MATH.bits() != 0 { CachePolicy::WriteCombining } else { CachePolicy::Writeback @@ -244,73 +225,96 @@ impl PageTableEntryTrait for PageTableEntry { } } - fn set_prop(&mut self, prop: PageProperty) { - let mut flags = PageTableFlags::VALID.bits() + fn pt_flags(&self) -> PageTableFlags { + let bits = PageTableFlags::empty().bits() as usize + | parse_flags!(self.0, PteFlags::RSV1, PageTableFlags::AVAIL1) + | parse_flags!(self.0, PteFlags::RSV2, PageTableFlags::AVAIL2); + PageTableFlags::from_bits(bits as u8).unwrap() + } + + fn new_page(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self { + let mut flags = PteFlags::VALID.bits() // FIXME: To avoid the PageModifyFault exception, // we set the DIRTY bit to 1 all the time. - | PageTableFlags::DIRTY.bits() + | PteFlags::DIRTY.bits() | parse_flags!( !prop.flags.bits(), PageFlags::R, - PageTableFlags::NOT_READABLE + PteFlags::NOT_READABLE ) - | parse_flags!(prop.flags.bits(), PageFlags::W, PageTableFlags::WRITABLE) + | parse_flags!(prop.flags.bits(), PageFlags::W, PteFlags::WRITABLE) | parse_flags!( !prop.flags.bits(), PageFlags::X, - PageTableFlags::NOT_EXECUTABLE + PteFlags::NOT_EXECUTABLE ) - | parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PageTableFlags::DIRTY) + | parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PteFlags::DIRTY) // TODO: How to get the accessed bit in loongarch? - | parse_flags!(prop.flags.bits(), PageFlags::ACCESSED, PageTableFlags::PRESENT) - | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PageTableFlags::RSV2); - flags |= parse_flags!( - prop.priv_flags.bits(), - PrivFlags::AVAIL1, - PageTableFlags::RSV1 - ); + | parse_flags!(prop.flags.bits(), PageFlags::ACCESSED, PteFlags::PRESENT) + | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PteFlags::RSV2); + flags |= parse_flags!(prop.priv_flags.bits(), PrivFlags::AVAIL1, PteFlags::RSV1); if prop.priv_flags.contains(PrivFlags::USER) { - flags |= PageTableFlags::PLVL.bits(); - flags |= PageTableFlags::PLVH.bits(); + flags |= PteFlags::PLVL.bits(); + flags |= PteFlags::PLVH.bits(); } if prop.priv_flags.contains(PrivFlags::GLOBAL) { - if self.is_huge() { - flags |= PageTableFlags::GLOBAL_IN_HUGE.bits(); + if level != 1 { + flags |= PteFlags::GLOBAL_IN_HUGE.bits(); } else { - flags |= PageTableFlags::GLOBAL_OR_HUGE.bits(); + flags |= PteFlags::GLOBAL_OR_HUGE.bits(); } } match prop.cache { CachePolicy::Writeback => { - flags |= PageTableFlags::MATL.bits(); + flags |= PteFlags::MATL.bits(); } CachePolicy::Uncacheable => (), CachePolicy::WriteCombining => { - flags |= PageTableFlags::MATH.bits(); + flags |= PteFlags::MATH.bits(); } _ => panic!("unsupported cache policy"), } - - self.0 = (self.0 & Self::PHYS_ADDR_MASK) | flags; + let level_bits = if level != 1 { + PteFlags::GLOBAL_OR_HUGE.bits() + } else { + PteFlags::IS_BASIC.bits() + }; + Self((paddr & Self::PHYS_ADDR_MASK) | flags | level_bits) } - fn is_last(&self, level: PagingLevel) -> bool { - level == 1 || self.is_huge() + fn new_pt(paddr: Paddr, flags: PageTableFlags) -> Self { + let flags = PteFlags::VALID.bits() + | parse_flags!(flags.bits(), PageTableFlags::AVAIL1, PteFlags::RSV1) + | parse_flags!(flags.bits(), PageTableFlags::AVAIL2, PteFlags::RSV2); + Self(paddr & Self::PHYS_ADDR_MASK | flags) } } -impl fmt::Debug for PageTableEntry { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut f = f.debug_struct("PageTableEntry"); - f.field("raw", &format_args!("{:#x}", self.0)) - .field("paddr", &format_args!("{:#x}", self.paddr())) - .field("present", &self.is_present()) - .field( - "flags", - &PageTableFlags::from_bits_truncate(self.0 & !Self::PHYS_ADDR_MASK), - ) - .field("prop", &self.prop()) - .finish() +impl PodOnce for PageTableEntry {} + +/// SAFETY: The implementation is safe because: +/// - `from_usize` and `into_usize` are not overridden; +/// - `from_repr` and `repr` are correctly implemented; +/// - a zeroed PTE represents an absent entry. +unsafe impl PteTrait for PageTableEntry { + fn from_repr(repr: &PteScalar, level: PagingLevel) -> Self { + match repr { + PteScalar::Absent => PageTableEntry(0), + PteScalar::PageTable(paddr, flags) => Self::new_pt(*paddr, *flags), + PteScalar::Mapped(paddr, prop) => Self::new_page(*paddr, level, *prop), + } + } + + fn to_repr(&self, level: PagingLevel) -> PteScalar { + if self.0 & PteFlags::VALID.bits() == 0 { + return PteScalar::Absent; + } + + if self.is_last(level) { + PteScalar::Mapped(self.paddr(), self.prop()) + } else { + PteScalar::PageTable(self.paddr(), self.pt_flags()) + } } } diff --git a/ostd/src/arch/riscv/mm/mod.rs b/ostd/src/arch/riscv/mm/mod.rs index 49185e1f3..eb5908f07 100644 --- a/ostd/src/arch/riscv/mm/mod.rs +++ b/ostd/src/arch/riscv/mm/mod.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -use alloc::fmt; use core::ops::Range; use spin::Once; @@ -17,8 +16,10 @@ use crate::{ mm::{ PAGE_SIZE, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, dma::DmaDirection, - page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, - page_table::PageTableEntryTrait, + page_prop::{ + CachePolicy, PageFlags, PageProperty, PageTableFlags, PrivilegedPageFlags as PrivFlags, + }, + page_table::{PteScalar, PteTrait}, }, }; @@ -51,7 +52,7 @@ bitflags::bitflags! { #[derive(Pod)] #[repr(C)] /// Possible flags for a page table entry. - pub(crate) struct PageTableFlags: usize { + pub(crate) struct PteFlags: usize { /// Specifies whether the mapped frame or page table is valid. const VALID = 1 << 0; /// Controls whether reads to the mapped frames are allowed. @@ -151,10 +152,6 @@ pub(crate) unsafe fn sync_dma_range(range: Range) { unsafe { core::arch::asm!("fence rw, rw", options(nostack)) }; } -#[derive(Clone, Copy, Pod, Default)] -#[repr(C)] -pub(crate) struct PageTableEntry(usize); - /// Activates the given root-level page table. /// /// "satp" register doesn't have a field that encodes the cache policy, @@ -182,59 +179,47 @@ pub(crate) fn current_page_table_paddr() -> Paddr { riscv::register::satp::read().ppn() << 12 } -impl PageTableEntry { - const PHYS_ADDR_MASK: usize = 0x003F_FFFF_FFFF_FC00; +#[derive(Debug, Clone, Copy, Pod, Default)] +#[repr(C)] +pub(crate) struct PageTableEntry(usize); - fn new_paddr(paddr: Paddr) -> Self { - let ppn = paddr >> 12; - Self(ppn << 10) - } -} - -/// Parse a bit-flag bits `val` in the representation of `from` to `to` in bits. +/// Parses a bit-flag bits `val` in the representation of `from` to `to` in bits. macro_rules! parse_flags { ($val:expr, $from:expr, $to:expr) => { (($val as usize & $from.bits() as usize) >> $from.bits().ilog2() << $to.bits().ilog2()) }; } -impl PodOnce for PageTableEntry {} +impl PageTableEntry { + const PHYS_ADDR_MASK: usize = 0x003f_ffff_ffff_fc00; -impl PageTableEntryTrait for PageTableEntry { - fn is_present(&self) -> bool { - self.0 & PageTableFlags::VALID.bits() != 0 - } - - fn new_page(paddr: Paddr, _level: PagingLevel, prop: PageProperty) -> Self { - let mut pte = Self::new_paddr(paddr); - pte.set_prop(prop); - pte - } - - fn new_pt(paddr: Paddr) -> Self { - // In RISC-V, non-leaf PTE should have RWX = 000, - // and D, A, and U are reserved for future standard use. - let pte = Self::new_paddr(paddr); - PageTableEntry(pte.0 | PageTableFlags::VALID.bits()) + fn new_without_flags(paddr: Paddr) -> Self { + assert_eq!(paddr & !Self::PHYS_ADDR_MASK, 0); + Self(paddr >> 12 << 10) } fn paddr(&self) -> Paddr { - let ppn = (self.0 & Self::PHYS_ADDR_MASK) >> 10; - ppn << 12 + (self.0 & Self::PHYS_ADDR_MASK) >> 10 << 12 + } + + fn is_last(&self, level: PagingLevel) -> bool { + let rwx = PteFlags::READABLE | PteFlags::WRITABLE | PteFlags::EXECUTABLE; + level == 1 || (self.0 & rwx.bits()) != 0 } fn prop(&self) -> PageProperty { - let flags = (parse_flags!(self.0, PageTableFlags::READABLE, PageFlags::R)) - | (parse_flags!(self.0, PageTableFlags::WRITABLE, PageFlags::W)) - | (parse_flags!(self.0, PageTableFlags::EXECUTABLE, PageFlags::X)) - | (parse_flags!(self.0, PageTableFlags::ACCESSED, PageFlags::ACCESSED)) - | (parse_flags!(self.0, PageTableFlags::DIRTY, PageFlags::DIRTY)) - | (parse_flags!(self.0, PageTableFlags::RSV2, PageFlags::AVAIL2)); - let priv_flags = (parse_flags!(self.0, PageTableFlags::USER, PrivFlags::USER)) - | (parse_flags!(self.0, PageTableFlags::GLOBAL, PrivFlags::GLOBAL)) - | (parse_flags!(self.0, PageTableFlags::RSV1, PrivFlags::AVAIL1)); + let flags = parse_flags!(self.0, PteFlags::READABLE, PageFlags::R) + | parse_flags!(self.0, PteFlags::WRITABLE, PageFlags::W) + | parse_flags!(self.0, PteFlags::EXECUTABLE, PageFlags::X) + | parse_flags!(self.0, PteFlags::ACCESSED, PageFlags::ACCESSED) + | parse_flags!(self.0, PteFlags::DIRTY, PageFlags::DIRTY) + | parse_flags!(self.0, PteFlags::RSV2, PageFlags::AVAIL2); - let cache = if self.0 & PageTableFlags::PBMT_IO.bits() != 0 { + let priv_flags = parse_flags!(self.0, PteFlags::USER, PrivFlags::USER) + | parse_flags!(self.0, PteFlags::GLOBAL, PrivFlags::GLOBAL) + | parse_flags!(self.0, PteFlags::RSV1, PrivFlags::AVAIL1); + + let cache = if self.0 & PteFlags::PBMT_IO.bits() != 0 { CachePolicy::Uncacheable } else { CachePolicy::Writeback @@ -247,33 +232,24 @@ impl PageTableEntryTrait for PageTableEntry { } } - fn set_prop(&mut self, prop: PageProperty) { - let mut flags = PageTableFlags::VALID.bits() - | parse_flags!(prop.flags.bits(), PageFlags::R, PageTableFlags::READABLE) - | parse_flags!(prop.flags.bits(), PageFlags::W, PageTableFlags::WRITABLE) - | parse_flags!(prop.flags.bits(), PageFlags::X, PageTableFlags::EXECUTABLE) - | parse_flags!( - prop.flags.bits(), - PageFlags::ACCESSED, - PageTableFlags::ACCESSED - ) - | parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PageTableFlags::DIRTY) - | parse_flags!( - prop.priv_flags.bits(), - PrivFlags::USER, - PageTableFlags::USER - ) - | parse_flags!( - prop.priv_flags.bits(), - PrivFlags::GLOBAL, - PageTableFlags::GLOBAL - ) - | parse_flags!( - prop.priv_flags.bits(), - PrivFlags::AVAIL1, - PageTableFlags::RSV1 - ) - | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PageTableFlags::RSV2); + fn pt_flags(&self) -> PageTableFlags { + let bits = PageTableFlags::empty().bits() as usize + | parse_flags!(self.0, PteFlags::RSV1, PageTableFlags::AVAIL1) + | parse_flags!(self.0, PteFlags::RSV2, PageTableFlags::AVAIL2); + PageTableFlags::from_bits(bits as u8).unwrap() + } + + fn new_page(paddr: Paddr, _level: PagingLevel, prop: PageProperty) -> Self { + let mut flags = PteFlags::VALID.bits() + | parse_flags!(prop.flags.bits(), PageFlags::R, PteFlags::READABLE) + | parse_flags!(prop.flags.bits(), PageFlags::W, PteFlags::WRITABLE) + | parse_flags!(prop.flags.bits(), PageFlags::X, PteFlags::EXECUTABLE) + | parse_flags!(prop.flags.bits(), PageFlags::ACCESSED, PteFlags::ACCESSED) + | parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PteFlags::DIRTY) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::USER, PteFlags::USER) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::GLOBAL, PteFlags::GLOBAL) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::AVAIL1, PteFlags::RSV1) + | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PteFlags::RSV2); match prop.cache { CachePolicy::Writeback => (), @@ -282,32 +258,52 @@ impl PageTableEntryTrait for PageTableEntry { // memory. Normal memory can also be `Noncacheable`, where the // PBMT should be set to `PBMT_NC`. if has_extensions(IsaExtensions::SVPBMT) { - flags |= PageTableFlags::PBMT_IO.bits() + flags |= PteFlags::PBMT_IO.bits() } } _ => panic!("unsupported cache policy"), } - self.0 = (self.0 & Self::PHYS_ADDR_MASK) | flags; + let res = Self::new_without_flags(paddr); + Self(res.0 | flags) } - fn is_last(&self, level: PagingLevel) -> bool { - let rwx = PageTableFlags::READABLE | PageTableFlags::WRITABLE | PageTableFlags::EXECUTABLE; - level == 1 || (self.0 & rwx.bits()) != 0 + fn new_pt(paddr: Paddr, flags: PageTableFlags) -> Self { + // In RISC-V, non-leaf PTE should have RWX = 000, + // and D, A, and U are reserved for future standard use. + let flags = PteFlags::VALID.bits() + | parse_flags!(flags.bits(), PageTableFlags::AVAIL1, PteFlags::RSV1) + | parse_flags!(flags.bits(), PageTableFlags::AVAIL2, PteFlags::RSV2); + + let res = Self::new_without_flags(paddr); + Self(res.0 | flags) } } -impl fmt::Debug for PageTableEntry { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut f = f.debug_struct("PageTableEntry"); - f.field("raw", &format_args!("{:#x}", self.0)) - .field("paddr", &format_args!("{:#x}", self.paddr())) - .field("present", &self.is_present()) - .field( - "flags", - &PageTableFlags::from_bits_truncate(self.0 & !Self::PHYS_ADDR_MASK), - ) - .field("prop", &self.prop()) - .finish() +impl PodOnce for PageTableEntry {} + +/// SAFETY: The implementation is safe because: +/// - `from_usize` and `into_usize` are not overridden; +/// - `from_repr` and `repr` are correctly implemented; +/// - a zeroed PTE represents an absent entry. +unsafe impl PteTrait for PageTableEntry { + fn from_repr(repr: &PteScalar, level: PagingLevel) -> Self { + match repr { + PteScalar::Absent => PageTableEntry(0), + PteScalar::PageTable(paddr, flags) => Self::new_pt(*paddr, *flags), + PteScalar::Mapped(paddr, prop) => Self::new_page(*paddr, level, *prop), + } + } + + fn to_repr(&self, level: PagingLevel) -> PteScalar { + if self.0 & PteFlags::VALID.bits() == 0 { + return PteScalar::Absent; + } + + if self.is_last(level) { + PteScalar::Mapped(self.paddr(), self.prop()) + } else { + PteScalar::PageTable(self.paddr(), self.pt_flags()) + } } } diff --git a/ostd/src/arch/x86/iommu/dma_remapping/second_stage.rs b/ostd/src/arch/x86/iommu/dma_remapping/second_stage.rs index 4a5535681..83879285b 100644 --- a/ostd/src/arch/x86/iommu/dma_remapping/second_stage.rs +++ b/ostd/src/arch/x86/iommu/dma_remapping/second_stage.rs @@ -1,15 +1,13 @@ // SPDX-License-Identifier: MPL-2.0 -#![expect(unused_variables)] - use core::ops::Range; use crate::{ Pod, mm::{ Paddr, PageProperty, PagingConstsTrait, PagingLevel, PodOnce, - page_prop::{CachePolicy, PageFlags, PrivilegedPageFlags as PrivFlags}, - page_table::{PageTableConfig, PageTableEntryTrait}, + page_prop::{CachePolicy, PageFlags, PageTableFlags, PrivilegedPageFlags as PrivFlags}, + page_table::{PageTableConfig, PteScalar, PteTrait}, }, }; @@ -54,19 +52,20 @@ impl PagingConstsTrait for PagingConsts { bitflags::bitflags! { #[derive(Pod)] #[repr(C)] - pub struct PageTableFlags : u64{ + pub struct PteFlags: usize { /// Whether accesses to this page must snoop processor caches. const SNOOP = 1 << 11; - const DIRTY = 1 << 9; + /// Bits ignored by hardware. + const IGN2 = 1 << 9; + const IGN1 = 1 << 8; - const ACCESSED = 1 << 8; /// Whether this page table entry is the last entry. const LAST_PAGE = 1 << 7; /// Ignore PAT, 1 if the scalable-mode PASID-table entry is not /// used for effective memory-type determination. - const IGNORE_PAT = 1 << 6; + const IGNORE_PAT = 1 << 6; /// Extended Memory Type, ignored by hardware when the /// Extended Memory Type Enable (EMTE) field is Clear. @@ -84,81 +83,102 @@ bitflags::bitflags! { #[derive(Debug, Clone, Copy, Pod, Default)] #[repr(C)] -pub struct PageTableEntry(u64); +pub struct PageTableEntry(usize); -impl PageTableEntry { - const PHYS_MASK: u64 = 0xFFFF_FFFF_F000; - const PROP_MASK: u64 = !Self::PHYS_MASK & !PageTableFlags::LAST_PAGE.bits(); +/// Parses a bit-flag bits `val` in the representation of `from` to `to` in bits. +macro_rules! parse_flags { + ($val:expr, $from:expr, $to:expr) => { + (($val as usize & $from.bits() as usize) >> $from.bits().ilog2() << $to.bits().ilog2()) + }; } -impl PodOnce for PageTableEntry {} - -impl PageTableEntryTrait for PageTableEntry { - fn new_page(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self { - let mut pte = Self(paddr as u64 & Self::PHYS_MASK | PageTableFlags::LAST_PAGE.bits()); - pte.set_prop(prop); - pte - } - - fn new_pt(paddr: Paddr) -> Self { - Self( - paddr as u64 & Self::PHYS_MASK - | PageTableFlags::READABLE.bits() - | PageTableFlags::WRITABLE.bits(), - ) - } - - fn paddr(&self) -> Paddr { - (self.0 & Self::PHYS_MASK) as usize - } +impl PageTableEntry { + const PHYS_MASK: usize = 0xffff_ffff_f000; fn is_present(&self) -> bool { - self.0 & (PageTableFlags::READABLE | PageTableFlags::WRITABLE).bits() != 0 + self.0 & (PteFlags::READABLE | PteFlags::LAST_PAGE).bits() != 0 + } + + fn is_last(&self, level: PagingLevel) -> bool { + level == 1 } fn prop(&self) -> PageProperty { - let mut flags = PageFlags::empty(); - if self.0 & PageTableFlags::READABLE.bits() != 0 { - flags |= PageFlags::R; - } - if self.0 & PageTableFlags::WRITABLE.bits() != 0 { - flags |= PageFlags::W; - } - if self.0 & PageTableFlags::ACCESSED.bits() != 0 { - flags |= PageFlags::ACCESSED; - } - if self.0 & PageTableFlags::DIRTY.bits() != 0 { - flags |= PageFlags::DIRTY; - } + let flags = parse_flags!(self.0, PteFlags::READABLE, PageFlags::R) + | parse_flags!(self.0, PteFlags::WRITABLE, PageFlags::W) + | parse_flags!(self.0, PteFlags::IGN2, PageFlags::AVAIL2); + + let priv_flags = parse_flags!(self.0, PteFlags::IGN1, PrivFlags::AVAIL1); + // TODO: The determination cache policy is not rigorous. We should revise it. - let cache = if self.0 & PageTableFlags::SNOOP.bits() != 0 { + let cache = if self.0 & PteFlags::SNOOP.bits() != 0 { CachePolicy::Writeback } else { CachePolicy::Uncacheable }; PageProperty { - flags, + flags: PageFlags::from_bits(flags as u8).unwrap(), cache, - priv_flags: PrivFlags::empty(), + priv_flags: PrivFlags::from_bits(priv_flags as u8).unwrap(), } } - fn set_prop(&mut self, prop: PageProperty) { - let mut flags = PageTableFlags::empty(); - if prop.flags.contains(PageFlags::W) { - flags |= PageTableFlags::WRITABLE; - } - if prop.flags.contains(PageFlags::R) { - flags |= PageTableFlags::READABLE; - } + fn pt_flags(&self) -> PageTableFlags { + let bits = PageTableFlags::empty().bits() as usize + | parse_flags!(self.0, PteFlags::IGN1, PageTableFlags::AVAIL1) + | parse_flags!(self.0, PteFlags::IGN2, PageTableFlags::AVAIL2); + PageTableFlags::from_bits(bits as u8).unwrap() + } + + fn new_page(paddr: Paddr, _level: PagingLevel, prop: PageProperty) -> Self { + let mut flags = PteFlags::LAST_PAGE.bits() + | parse_flags!(prop.flags.bits(), PageFlags::R, PteFlags::READABLE) + | parse_flags!(prop.flags.bits(), PageFlags::W, PteFlags::WRITABLE) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::AVAIL1, PteFlags::IGN1) + | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PteFlags::IGN2); + if prop.cache != CachePolicy::Uncacheable { - flags |= PageTableFlags::SNOOP; + flags |= PteFlags::SNOOP.bits(); } - self.0 = self.0 & !Self::PROP_MASK | flags.bits(); + + Self(paddr & Self::PHYS_MASK | flags) } - fn is_last(&self, level: PagingLevel) -> bool { - level == 1 + fn new_pt(paddr: Paddr, flags: PageTableFlags) -> Self { + let flags = PteFlags::READABLE.bits() + | PteFlags::WRITABLE.bits() + | parse_flags!(flags.bits(), PageTableFlags::AVAIL1, PteFlags::IGN1) + | parse_flags!(flags.bits(), PageTableFlags::AVAIL2, PteFlags::IGN2); + Self(paddr & Self::PHYS_MASK | flags) + } +} + +impl PodOnce for PageTableEntry {} + +/// SAFETY: The implementation is safe because: +/// - `from_usize` and `into_usize` are not overridden; +/// - `from_repr` and `repr` are correctly implemented; +/// - a zeroed PTE represents an absent entry. +unsafe impl PteTrait for PageTableEntry { + fn from_repr(repr: &PteScalar, level: PagingLevel) -> Self { + match repr { + PteScalar::Absent => PageTableEntry(0), + PteScalar::PageTable(paddr, flags) => Self::new_pt(*paddr, *flags), + PteScalar::Mapped(paddr, prop) => Self::new_page(*paddr, level, *prop), + } + } + + fn to_repr(&self, level: PagingLevel) -> PteScalar { + if !self.is_present() { + return PteScalar::Absent; + } + + let paddr = self.0 & Self::PHYS_MASK; + if self.is_last(level) { + PteScalar::Mapped(paddr, self.prop()) + } else { + PteScalar::PageTable(paddr, self.pt_flags()) + } } } diff --git a/ostd/src/arch/x86/mm/mod.rs b/ostd/src/arch/x86/mm/mod.rs index 5fab78595..b136a1d01 100644 --- a/ostd/src/arch/x86/mm/mod.rs +++ b/ostd/src/arch/x86/mm/mod.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: MPL-2.0 -use alloc::fmt; use core::ops::Range; use cfg_if::cfg_if; @@ -19,8 +18,10 @@ use crate::{ mm::{ PAGE_SIZE, Paddr, PagingConstsTrait, PagingLevel, PodOnce, Vaddr, dma::DmaDirection, - page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags as PrivFlags}, - page_table::PageTableEntryTrait, + page_prop::{ + CachePolicy, PageFlags, PageProperty, PageTableFlags, PrivilegedPageFlags as PrivFlags, + }, + page_table::{PteScalar, PteTrait}, }, }; @@ -45,7 +46,7 @@ bitflags::bitflags! { #[derive(Pod)] #[repr(C)] /// Possible flags for a page table entry. - pub(crate) struct PageTableFlags: usize { + pub(crate) struct PteFlags: usize { /// Specifies whether the mapped frame or page table is loaded in memory. const PRESENT = 1 << 0; /// Controls whether writes to the mapped frames are allowed. @@ -135,10 +136,6 @@ pub(crate) unsafe fn sync_dma_range(_range: Range) { // Reference: , . } -#[derive(Clone, Copy, Pod, Default)] -#[repr(C)] -pub(crate) struct PageTableEntry(usize); - /// Activates the given root-level page table. /// /// The cache policy of the root page table node is controlled by `root_pt_cache`. @@ -173,71 +170,66 @@ pub(crate) fn current_page_table_paddr() -> Paddr { .as_u64() as Paddr } -impl PageTableEntry { - cfg_if! { - if #[cfg(feature = "cvm_guest")] { - const PHYS_ADDR_MASK: usize = 0x7_FFFF_FFFF_F000; - } else { - const PHYS_ADDR_MASK: usize = 0xF_FFFF_FFFF_F000; - } - } - const PROP_MASK: usize = !Self::PHYS_ADDR_MASK & !PageTableFlags::HUGE.bits(); -} +#[derive(Debug, Clone, Copy, Pod)] +#[repr(C)] +pub(crate) struct PageTableEntry(usize); -/// Parse a bit-flag bits `val` in the representation of `from` to `to` in bits. +/// Parses a bit-flag bits `val` in the representation of `from` to `to` in bits. macro_rules! parse_flags { ($val:expr, $from:expr, $to:expr) => { - ($val as usize & $from.bits() as usize) >> $from.bits().ilog2() << $to.bits().ilog2() + (($val as usize & $from.bits() as usize) >> $from.bits().ilog2() << $to.bits().ilog2()) }; } -impl PodOnce for PageTableEntry {} +impl PageTableEntry { + cfg_if! { + if #[cfg(feature = "cvm_guest")] { + const PHYS_ADDR_MASK_LVL1: usize = 0x7_ffff_ffff_f000; + const PHYS_ADDR_MASK_LVL2: usize = 0x7_ffff_ffe0_0000; + const PHYS_ADDR_MASK_LVL3: usize = 0x7_ffff_c000_0000; + } else { + const PHYS_ADDR_MASK_LVL1: usize = 0xf_ffff_ffff_f000; + const PHYS_ADDR_MASK_LVL2: usize = 0xf_ffff_ffe0_0000; + const PHYS_ADDR_MASK_LVL3: usize = 0xf_ffff_c000_0000; + } + } + + const CHILD_PT_ADDR_MASK: usize = Self::PHYS_ADDR_MASK_LVL1; + + fn pa_mask_at_level(level: PagingLevel) -> usize { + match level { + 1 => Self::PHYS_ADDR_MASK_LVL1, + 2 => Self::PHYS_ADDR_MASK_LVL2, + 3 => Self::PHYS_ADDR_MASK_LVL3, + _ => panic!("invalid level {} for page entry", level), + } + } -impl PageTableEntryTrait for PageTableEntry { fn is_present(&self) -> bool { // For PT child, `PRESENT` should be set; for huge page, `HUGE` should // be set; for the leaf child page, `PAT`, which is the same bit as // the `HUGE` bit in upper levels, should be set. - self.0 & PageTableFlags::PRESENT.bits() != 0 || self.0 & PageTableFlags::HUGE.bits() != 0 - } - - fn new_page(paddr: Paddr, _level: PagingLevel, prop: PageProperty) -> Self { - let flags = PageTableFlags::HUGE.bits(); - let mut pte = Self(paddr & Self::PHYS_ADDR_MASK | flags); - pte.set_prop(prop); - pte - } - - fn new_pt(paddr: Paddr) -> Self { - // In x86 if it's an intermediate PTE, it's better to have the same permissions - // as the most permissive child (to reduce hardware page walk accesses). But we - // don't have a mechanism to keep it generic across architectures, thus just - // setting it to be the most permissive. - let flags = PageTableFlags::PRESENT.bits() - | PageTableFlags::WRITABLE.bits() - | PageTableFlags::USER.bits(); - Self(paddr & Self::PHYS_ADDR_MASK | flags) - } - - fn paddr(&self) -> Paddr { - self.0 & Self::PHYS_ADDR_MASK + self.0 & PteFlags::PRESENT.bits() != 0 || self.0 & PteFlags::HUGE.bits() != 0 } fn prop(&self) -> PageProperty { - let flags = (parse_flags!(self.0, PageTableFlags::PRESENT, PageFlags::R)) - | (parse_flags!(self.0, PageTableFlags::WRITABLE, PageFlags::W)) - | (parse_flags!(!self.0, PageTableFlags::NO_EXECUTE, PageFlags::X)) - | (parse_flags!(self.0, PageTableFlags::ACCESSED, PageFlags::ACCESSED)) - | (parse_flags!(self.0, PageTableFlags::DIRTY, PageFlags::DIRTY)) - | (parse_flags!(self.0, PageTableFlags::HIGH_IGN2, PageFlags::AVAIL2)); - let priv_flags = (parse_flags!(self.0, PageTableFlags::USER, PrivFlags::USER)) - | (parse_flags!(self.0, PageTableFlags::GLOBAL, PrivFlags::GLOBAL)) - | (parse_flags!(self.0, PageTableFlags::HIGH_IGN1, PrivFlags::AVAIL1)); + let flags = parse_flags!(self.0, PteFlags::PRESENT, PageFlags::R) + | parse_flags!(self.0, PteFlags::WRITABLE, PageFlags::W) + | parse_flags!(!self.0, PteFlags::NO_EXECUTE, PageFlags::X) + | parse_flags!(self.0, PteFlags::ACCESSED, PageFlags::ACCESSED) + | parse_flags!(self.0, PteFlags::DIRTY, PageFlags::DIRTY) + | parse_flags!(self.0, PteFlags::HIGH_IGN2, PageFlags::AVAIL2); + + let priv_flags = parse_flags!(self.0, PteFlags::USER, PrivFlags::USER) + | parse_flags!(self.0, PteFlags::GLOBAL, PrivFlags::GLOBAL) + | parse_flags!(self.0, PteFlags::HIGH_IGN1, PrivFlags::AVAIL1); + #[cfg(feature = "cvm_guest")] - let priv_flags = - priv_flags | (parse_flags!(self.0, PageTableFlags::SHARED, PrivFlags::SHARED)); + let priv_flags = priv_flags | parse_flags!(self.0, PteFlags::SHARED, PrivFlags::SHARED); + // Determine cache policy from PCD, PWT bits. - let cache = flags_to_cache_policy(PageTableFlags::from_bits_truncate(self.0)); + let cache = flags_to_cache_policy(PteFlags::from_bits_truncate(self.0)); + PageProperty { flags: PageFlags::from_bits(flags as u8).unwrap(), cache, @@ -245,69 +237,91 @@ impl PageTableEntryTrait for PageTableEntry { } } - fn set_prop(&mut self, prop: PageProperty) { - if !self.is_present() { - return; - } - let mut flags = PageTableFlags::empty().bits(); - flags |= (parse_flags!(prop.flags.bits(), PageFlags::R, PageTableFlags::PRESENT)) - | (parse_flags!(prop.flags.bits(), PageFlags::W, PageTableFlags::WRITABLE)) - | (parse_flags!(!prop.flags.bits(), PageFlags::X, PageTableFlags::NO_EXECUTE)) - | (parse_flags!( - prop.flags.bits(), - PageFlags::ACCESSED, - PageTableFlags::ACCESSED - )) - | (parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PageTableFlags::DIRTY)) - | (parse_flags!( - prop.priv_flags.bits(), - PrivFlags::AVAIL1, - PageTableFlags::HIGH_IGN1 - )) - | (parse_flags!( - prop.flags.bits(), - PageFlags::AVAIL2, - PageTableFlags::HIGH_IGN2 - )) - | (parse_flags!( - prop.priv_flags.bits(), - PrivFlags::USER, - PageTableFlags::USER - )) - | (parse_flags!( - prop.priv_flags.bits(), - PrivFlags::GLOBAL, - PageTableFlags::GLOBAL - )); - #[cfg(feature = "cvm_guest")] - { - flags |= parse_flags!( - prop.priv_flags.bits(), - PrivFlags::SHARED, - PageTableFlags::SHARED - ); - } - flags |= cache_policy_to_flags(prop.cache).bits(); - self.0 = self.0 & !Self::PROP_MASK | flags; + fn pt_flags(&self) -> PageTableFlags { + let bits = PageTableFlags::empty().bits() as usize + | parse_flags!(self.0, PteFlags::HIGH_IGN1, PageTableFlags::AVAIL1) + | parse_flags!(self.0, PteFlags::HIGH_IGN2, PageTableFlags::AVAIL2); + PageTableFlags::from_bits(bits as u8).unwrap() } - fn is_last(&self, _level: PagingLevel) -> bool { - self.0 & PageTableFlags::HUGE.bits() != 0 + fn new_page(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self { + let mut flags = PteFlags::HUGE.bits(); + + flags |= parse_flags!(prop.flags.bits(), PageFlags::R, PteFlags::PRESENT) + | parse_flags!(prop.flags.bits(), PageFlags::W, PteFlags::WRITABLE) + | parse_flags!(!prop.flags.bits(), PageFlags::X, PteFlags::NO_EXECUTE) + | parse_flags!(prop.flags.bits(), PageFlags::ACCESSED, PteFlags::ACCESSED) + | parse_flags!(prop.flags.bits(), PageFlags::DIRTY, PteFlags::DIRTY) + | parse_flags!( + prop.priv_flags.bits(), + PrivFlags::AVAIL1, + PteFlags::HIGH_IGN1 + ) + | parse_flags!(prop.flags.bits(), PageFlags::AVAIL2, PteFlags::HIGH_IGN2) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::USER, PteFlags::USER) + | parse_flags!(prop.priv_flags.bits(), PrivFlags::GLOBAL, PteFlags::GLOBAL); + #[cfg(feature = "cvm_guest")] + { + flags |= parse_flags!(prop.priv_flags.bits(), PrivFlags::SHARED, PteFlags::SHARED); + } + + flags |= cache_policy_to_flags(prop.cache).bits(); + + assert_eq!( + paddr & !Self::pa_mask_at_level(level), + 0, + "page physical address contains invalid bits" + ); + Self(paddr | flags) + } + + fn new_pt(paddr: Paddr, flags: PageTableFlags) -> Self { + // In x86 if it's an intermediate PTE, it's better to have the same permissions + // as the most permissive child (to reduce hardware page walk accesses). But we + // don't have a mechanism to keep it generic across architectures, thus just + // setting it to be the most permissive. + let flags = PteFlags::PRESENT.bits() + | PteFlags::WRITABLE.bits() + | PteFlags::USER.bits() + | parse_flags!(flags.bits(), PageTableFlags::AVAIL1, PteFlags::HIGH_IGN1) + | parse_flags!(flags.bits(), PageTableFlags::AVAIL2, PteFlags::HIGH_IGN2); + + assert_eq!( + paddr & !Self::CHILD_PT_ADDR_MASK, + 0, + "page table physical address contains invalid bits" + ); + Self(paddr | flags) } } -impl fmt::Debug for PageTableEntry { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let mut f = f.debug_struct("PageTableEntry"); - f.field("raw", &format_args!("{:#x}", self.0)) - .field("paddr", &format_args!("{:#x}", self.paddr())) - .field("present", &self.is_present()) - .field( - "flags", - &PageTableFlags::from_bits_truncate(self.0 & !Self::PHYS_ADDR_MASK), - ) - .field("prop", &self.prop()) - .finish() +impl PodOnce for PageTableEntry {} + +/// SAFETY: The implementation is safe because: +/// - `from_usize` and `into_usize` are not overridden; +/// - `from_repr` and `repr` are correctly implemented; +/// - a zeroed PTE represents an absent entry. +unsafe impl PteTrait for PageTableEntry { + fn from_repr(repr: &PteScalar, level: PagingLevel) -> Self { + match repr { + PteScalar::Absent => PageTableEntry(0), + PteScalar::PageTable(paddr, flags) => Self::new_pt(*paddr, *flags), + PteScalar::Mapped(paddr, prop) => Self::new_page(*paddr, level, *prop), + } + } + + fn to_repr(&self, level: PagingLevel) -> PteScalar { + if !self.is_present() { + return PteScalar::Absent; + } + + if self.0 & PteFlags::HUGE.bits() != 0 { + let paddr = self.0 & Self::pa_mask_at_level(level); + PteScalar::Mapped(paddr, self.prop()) + } else { + let paddr = self.0 & Self::CHILD_PT_ADDR_MASK; + PteScalar::PageTable(paddr, self.pt_flags()) + } } } diff --git a/ostd/src/arch/x86/mm/pat.rs b/ostd/src/arch/x86/mm/pat.rs index 47df9586b..9bcb29856 100644 --- a/ostd/src/arch/x86/mm/pat.rs +++ b/ostd/src/arch/x86/mm/pat.rs @@ -2,7 +2,7 @@ use x86::msr::{IA32_PAT, wrmsr}; -use super::PageTableFlags; +use super::PteFlags; use crate::{const_assert, mm::page_prop::CachePolicy}; /// Software-defined mapping from PAT (page attribute table) bit combinations @@ -23,29 +23,27 @@ const IA32_PAT_MAPPINGS: [CachePolicy; 8] = [ CachePolicy::Uncacheable, // Index 7: PAT=1, PCD=1, PWT=1 (same as 3) ]; -pub(super) const fn flags_to_cache_policy(flags: PageTableFlags) -> CachePolicy { +pub(super) const fn flags_to_cache_policy(flags: PteFlags) -> CachePolicy { let bits = flags.bits(); let mut index = 0usize; - if bits & PageTableFlags::NO_CACHE.bits() != 0 { + if bits & PteFlags::NO_CACHE.bits() != 0 { index |= 2; } - if bits & PageTableFlags::WRITE_THROUGH.bits() != 0 { + if bits & PteFlags::WRITE_THROUGH.bits() != 0 { index |= 1; } IA32_PAT_MAPPINGS[index] } -pub(super) const fn cache_policy_to_flags(cache_policy: CachePolicy) -> PageTableFlags { +pub(super) const fn cache_policy_to_flags(cache_policy: CachePolicy) -> PteFlags { let bits = match cache_policy { CachePolicy::Writeback => 0, - CachePolicy::Writethrough => PageTableFlags::WRITE_THROUGH.bits(), - CachePolicy::Uncacheable => { - PageTableFlags::NO_CACHE.bits() | PageTableFlags::WRITE_THROUGH.bits() - } - CachePolicy::WriteCombining => PageTableFlags::NO_CACHE.bits(), + CachePolicy::Writethrough => PteFlags::WRITE_THROUGH.bits(), + CachePolicy::Uncacheable => PteFlags::NO_CACHE.bits() | PteFlags::WRITE_THROUGH.bits(), + CachePolicy::WriteCombining => PteFlags::NO_CACHE.bits(), _ => panic!("unsupported cache policy"), }; - PageTableFlags::from_bits_truncate(bits) + PteFlags::from_bits_truncate(bits) } const_assert!(matches!( diff --git a/ostd/src/mm/frame/meta.rs b/ostd/src/mm/frame/meta.rs index ba4d7fcb6..413944583 100644 --- a/ostd/src/mm/frame/meta.rs +++ b/ostd/src/mm/frame/meta.rs @@ -490,7 +490,7 @@ pub(crate) unsafe fn init() -> Segment { priv_flags: PrivilegedPageFlags::GLOBAL, }; // SAFETY: we are doing the metadata mappings for the kernel. - unsafe { boot_pt.map_base_page(vaddr, frame_paddr / PAGE_SIZE, prop) }; + unsafe { boot_pt.map_base_page(vaddr, frame_paddr, prop) }; } }) .unwrap(); @@ -635,7 +635,7 @@ fn add_temp_linear_mapping(max_paddr: Paddr) { boot_pt::with_borrow(|boot_pt| { for paddr in prange.step_by(PAGE_SIZE) { let vaddr = LINEAR_MAPPING_BASE_VADDR + paddr; - boot_pt.map_base_page(vaddr, paddr / PAGE_SIZE, prop); + boot_pt.map_base_page(vaddr, paddr, prop); } }) .unwrap(); diff --git a/ostd/src/mm/mod.rs b/ostd/src/mm/mod.rs index 4ff9256bf..a5711a2b2 100644 --- a/ostd/src/mm/mod.rs +++ b/ostd/src/mm/mod.rs @@ -42,7 +42,9 @@ pub use self::{ vm_space::VmSpace, }; pub(crate) use self::{ - kspace::paddr_to_vaddr, page_prop::PrivilegedPageFlags, page_table::PageTable, + kspace::paddr_to_vaddr, + page_prop::{PageTableFlags, PrivilegedPageFlags}, + page_table::PageTable, }; use crate::arch::mm::PagingConsts; diff --git a/ostd/src/mm/page_prop.rs b/ostd/src/mm/page_prop.rs index abbfe8977..0f59c9e17 100644 --- a/ostd/src/mm/page_prop.rs +++ b/ostd/src/mm/page_prop.rs @@ -132,3 +132,13 @@ bitflags! { const SHARED = 0b10000000; } } + +bitflags! { + /// Flags that can be stored on intermediate page table entries. + pub(crate) struct PageTableFlags: u8 { + /// The first bit available for software use. + const AVAIL1 = 0b01000000; + /// The second bit available for software use. + const AVAIL2 = 0b10000000; + } +} diff --git a/ostd/src/mm/page_table/boot_pt.rs b/ostd/src/mm/page_table/boot_pt.rs index 3cb5eedf5..c544a3870 100644 --- a/ostd/src/mm/page_table/boot_pt.rs +++ b/ostd/src/mm/page_table/boot_pt.rs @@ -10,25 +10,27 @@ use core::{ sync::atomic::{AtomicU32, Ordering}, }; -use super::{PageTableEntryTrait, pte_index}; +use ostd_pod::Pod; + +use super::{PteTrait, pte_index}; use crate::{ arch::mm::{PageTableEntry, PagingConsts}, cpu::num_cpus, cpu_local_cell, mm::{ Frame, FrameAllocOptions, PAGE_SIZE, Paddr, PageProperty, PagingConstsTrait, PagingLevel, - PrivilegedPageFlags, Vaddr, + Vaddr, frame::{ self, allocator::{self, EarlyAllocatedFrameMeta}, }, nr_subpage_per_huge, paddr_to_vaddr, + page_prop::PageTableFlags, + page_table::PteScalar, }, sync::SpinLock, }; -type FrameNumber = usize; - /// The accessor to the boot page table singleton [`BootPageTable`]. /// /// The user should provide a closure to access the boot page table. The @@ -77,14 +79,14 @@ pub(crate) unsafe fn dismiss() { dfs_walk_on_leave::( boot_pt.root_pt, PagingConsts::NR_LEVELS, - &mut |pte| { - if !pte.prop().priv_flags.contains(PTE_POINTS_TO_FIRMWARE_PT) { + &mut |pte, pa, _, flags| { + if !flags.contains(PTE_POINTS_TO_FIRMWARE_PT) { // SAFETY: The pointed frame is allocated and forgotten with `into_raw`. - drop(unsafe { Frame::::from_raw(pte.paddr()) }) + drop(unsafe { Frame::::from_raw(pa) }) } // Firmware provided page tables may be a DAG instead of a tree. // Clear it to avoid double-free when we meet it the second time. - *pte = PageTableEntry::new_absent(); + *pte = PageTableEntry::new_zeroed(); }, ); } @@ -107,12 +109,9 @@ cpu_local_cell! { /// All the newly allocated page table frames have the first unused bit in /// parent PTEs. This allows us to deallocate them when the boot page table /// is dropped. -pub(crate) struct BootPageTable< - E: PageTableEntryTrait = PageTableEntry, - C: PagingConstsTrait = PagingConsts, -> { - root_pt: FrameNumber, - _pretend_to_use: core::marker::PhantomData<(E, C)>, +pub(crate) struct BootPageTable { + root_pt: Paddr, + _phantom: core::marker::PhantomData<(E, C)>, } // We use extra two available bits in the boot PT for memory management. @@ -120,9 +119,9 @@ pub(crate) struct BootPageTable< // The first available bit is used to differentiate firmware page tables from // the page tables allocated here. The second is for identifying double-visits // when walking the page tables since the PT can be a DAG. -const PTE_POINTS_TO_FIRMWARE_PT: PrivilegedPageFlags = PrivilegedPageFlags::AVAIL1; +const PTE_POINTS_TO_FIRMWARE_PT: PageTableFlags = PageTableFlags::AVAIL1; -impl BootPageTable { +impl BootPageTable { /// Creates a new boot page table from the current page table root /// physical address. /// @@ -132,22 +131,25 @@ impl BootPageTable { /// Otherwise, It would lead to double-drop of the page table frames set up /// by the firmware, loader or the setup code. unsafe fn from_current_pt() -> Self { - let root_pt = crate::arch::mm::current_page_table_paddr() / C::BASE_PAGE_SIZE; + let root_pt = crate::arch::mm::current_page_table_paddr(); // Make sure the 2 available bits are not set for firmware page tables. - dfs_walk_on_leave::(root_pt, C::NR_LEVELS, &mut |pte: &mut E| { - let mut prop = pte.prop(); - prop.priv_flags |= PTE_POINTS_TO_FIRMWARE_PT; - pte.set_prop(prop); - }); + dfs_walk_on_leave::( + root_pt, + C::NR_LEVELS, + &mut |pte: &mut E, pa, level, mut flags| { + flags |= PTE_POINTS_TO_FIRMWARE_PT; + *pte = E::from_repr(&PteScalar::PageTable(pa, flags), level); + }, + ); Self { root_pt, - _pretend_to_use: core::marker::PhantomData, + _phantom: core::marker::PhantomData, } } /// Returns the root physical address of the boot page table. pub(crate) fn root_address(&self) -> Paddr { - self.root_pt * C::BASE_PAGE_SIZE + self.root_pt } /// Maps a base page to a frame. @@ -160,33 +162,37 @@ impl BootPageTable { /// /// This function is unsafe because it can cause undefined behavior if the caller /// maps a page in the kernel address space. - pub unsafe fn map_base_page(&mut self, from: Vaddr, to: FrameNumber, prop: PageProperty) { + pub unsafe fn map_base_page(&mut self, from: Vaddr, to: Paddr, prop: PageProperty) { let mut pt = self.root_pt; let mut level = C::NR_LEVELS; // Walk to the last level of the page table. while level > 1 { let index = pte_index::(from, level); - let pte_ptr = unsafe { (paddr_to_vaddr(pt * C::BASE_PAGE_SIZE) as *mut E).add(index) }; + let pte_ptr = unsafe { (paddr_to_vaddr(pt) as *mut E).add(index) }; let pte = unsafe { pte_ptr.read() }; - pt = if !pte.is_present() { - let pte = self.alloc_child(); - unsafe { pte_ptr.write(pte) }; - pte.paddr() / C::BASE_PAGE_SIZE - } else if pte.is_last(level) { - panic!("mapping an already mapped huge page in the boot page table"); - } else { - pte.paddr() / C::BASE_PAGE_SIZE + match pte.to_repr(level) { + PteScalar::Absent => { + let (pte, child_pt) = self.alloc_child(level); + unsafe { pte_ptr.write(pte) }; + pt = child_pt; + } + PteScalar::Mapped(_, _) => { + panic!("mapping an already mapped huge page in the boot page table"); + } + PteScalar::PageTable(child_pt, _) => { + pt = child_pt; + } }; level -= 1; } // Map the page in the last level page table. let index = pte_index::(from, 1); - let pte_ptr = unsafe { (paddr_to_vaddr(pt * C::BASE_PAGE_SIZE) as *mut E).add(index) }; + let pte_ptr = unsafe { (paddr_to_vaddr(pt) as *mut E).add(index) }; let pte = unsafe { pte_ptr.read() }; - if pte.is_present() { + if matches!(pte.to_repr(1), PteScalar::Mapped(_, _)) { panic!("mapping an already mapped page in the boot page table"); } - unsafe { pte_ptr.write(E::new_page(to * C::BASE_PAGE_SIZE, 1, prop)) }; + unsafe { pte_ptr.write(E::from_repr(&PteScalar::Mapped(to, prop), 1)) }; } /// Set protections of a base page mapping. @@ -213,45 +219,45 @@ impl BootPageTable { // Walk to the last level of the page table. while level > 1 { let index = pte_index::(virt_addr, level); - let pte_ptr = unsafe { (paddr_to_vaddr(pt * C::BASE_PAGE_SIZE) as *mut E).add(index) }; + let pte_ptr = unsafe { (paddr_to_vaddr(pt) as *mut E).add(index) }; let pte = unsafe { pte_ptr.read() }; - pt = if !pte.is_present() { - panic!("protecting an unmapped page in the boot page table"); - } else if pte.is_last(level) { - // Split the huge page. - let child_pte = self.alloc_child(); - let child_frame_pa = child_pte.paddr(); - let huge_pa = pte.paddr(); - for i in 0..nr_subpage_per_huge::() { - let nxt_ptr = unsafe { (paddr_to_vaddr(child_frame_pa) as *mut E).add(i) }; - unsafe { - nxt_ptr.write(E::new_page( - huge_pa + i * C::BASE_PAGE_SIZE, - level - 1, - pte.prop(), - )) - }; + match pte.to_repr(level) { + PteScalar::Absent => { + panic!("protecting an unmapped page in the boot page table"); + } + PteScalar::PageTable(child_pt, _) => { + pt = child_pt; + } + PteScalar::Mapped(huge_pa, prop) => { + // Split the huge page. + let (child_pte, child_frame_pa) = self.alloc_child(level); + for i in 0..nr_subpage_per_huge::() { + let nxt_ptr = unsafe { (paddr_to_vaddr(child_frame_pa) as *mut E).add(i) }; + unsafe { + nxt_ptr.write(E::from_repr( + &PteScalar::Mapped(huge_pa + i * C::BASE_PAGE_SIZE, prop), + level - 1, + )) + }; + } + unsafe { pte_ptr.write(child_pte) }; + pt = child_frame_pa; } - unsafe { pte_ptr.write(E::new_pt(child_frame_pa)) }; - child_frame_pa / C::BASE_PAGE_SIZE - } else { - pte.paddr() / C::BASE_PAGE_SIZE }; level -= 1; } // Do protection in the last level page table. let index = pte_index::(virt_addr, 1); - let pte_ptr = unsafe { (paddr_to_vaddr(pt * C::BASE_PAGE_SIZE) as *mut E).add(index) }; + let pte_ptr = unsafe { (paddr_to_vaddr(pt) as *mut E).add(index) }; let pte = unsafe { pte_ptr.read() }; - if !pte.is_present() { + let PteScalar::Mapped(pa, mut prop) = pte.to_repr(1) else { panic!("protecting an unmapped page in the boot page table"); - } - let mut prop = pte.prop(); + }; op(&mut prop); - unsafe { pte_ptr.write(E::new_page(pte.paddr(), 1, prop)) }; + unsafe { pte_ptr.write(E::from_repr(&PteScalar::Mapped(pa, prop), 1)) }; } - fn alloc_child(&mut self) -> E { + fn alloc_child(&mut self, level: PagingLevel) -> (E, Paddr) { let frame_paddr = if frame::meta::is_initialized() { let frame = FrameAllocOptions::new() .zeroed(false) @@ -269,14 +275,20 @@ impl BootPageTable { let vaddr = paddr_to_vaddr(frame_paddr) as *mut u8; unsafe { core::ptr::write_bytes(vaddr, 0, PAGE_SIZE) }; - E::new_pt(frame_paddr) + ( + E::from_repr( + &PteScalar::PageTable(frame_paddr, PageTableFlags::empty()), + level, + ), + frame_paddr, + ) } #[cfg(ktest)] - pub(super) fn new(root_pt: FrameNumber) -> Self { + pub(super) fn new(root_pt: Paddr) -> Self { Self { root_pt, - _pretend_to_use: core::marker::PhantomData, + _phantom: core::marker::PhantomData, } } } @@ -285,18 +297,18 @@ impl BootPageTable { /// /// Once leaving a page table frame, the closure will be called with the PTE to /// the frame. -fn dfs_walk_on_leave( - pt: FrameNumber, +fn dfs_walk_on_leave( + pt: Paddr, level: PagingLevel, - op: &mut impl FnMut(&mut E), + op: &mut impl FnMut(&mut E, Paddr, PagingLevel, PageTableFlags), ) { if level >= 2 { - let pt_vaddr = paddr_to_vaddr(pt * C::BASE_PAGE_SIZE) as *mut E; + let pt_vaddr = paddr_to_vaddr(pt) as *mut E; let pt = unsafe { core::slice::from_raw_parts_mut(pt_vaddr, nr_subpage_per_huge::()) }; for pte in pt { - if pte.is_present() && !pte.is_last(level) { - dfs_walk_on_leave::(pte.paddr() / C::BASE_PAGE_SIZE, level - 1, op); - op(pte) + if let PteScalar::PageTable(child_pt, flags) = pte.to_repr(level) { + dfs_walk_on_leave::(child_pt, level - 1, op); + op(pte, child_pt, level, flags); } } } diff --git a/ostd/src/mm/page_table/cursor/locking.rs b/ostd/src/mm/page_table/cursor/locking.rs index 5dd805610..3458d760a 100644 --- a/ostd/src/mm/page_table/cursor/locking.rs +++ b/ostd/src/mm/page_table/cursor/locking.rs @@ -11,8 +11,8 @@ use crate::{ mm::{ HasPaddr, Vaddr, nr_subpage_per_huge, paddr_to_vaddr, page_table::{ - ChildRef, PageTable, PageTableConfig, PageTableEntryTrait, PageTableGuard, - PageTableNodeRef, PagingConstsTrait, PagingLevel, load_pte, page_size, pte_index, + ChildRef, PageTable, PageTableConfig, PageTableGuard, PageTableNodeRef, + PagingConstsTrait, PagingLevel, PteScalar, PteTrait, load_pte, page_size, pte_index, }, }, task::atomic_mode::InAtomicMode, @@ -112,13 +112,16 @@ fn try_traverse_and_lock_subtree_root<'rcu, C: PageTableConfig>( // - All page table entries are aligned and accessed with atomic operations only. let cur_pte = unsafe { load_pte(cur_pt_ptr.add(start_idx), Ordering::Acquire) }; - if cur_pte.is_present() { - if cur_pte.is_last(cur_level) { + match cur_pte.to_repr(cur_level) { + PteScalar::Mapped(_, _) => { break; } - cur_pt_addr = cur_pte.paddr(); - cur_node_guard = None; - continue; + PteScalar::Absent => {} + PteScalar::PageTable(child_pt_addr, _) => { + cur_pt_addr = child_pt_addr; + cur_node_guard = None; + continue; + } } // In case the child is absent, we should lock and allocate a new page table node. @@ -133,18 +136,19 @@ fn try_traverse_and_lock_subtree_root<'rcu, C: PageTableConfig>( } let mut cur_entry = pt_guard.entry(start_idx); - if cur_entry.is_none() { - let allocated_guard = cur_entry.alloc_if_none(guard).unwrap(); - cur_pt_addr = allocated_guard.paddr(); - cur_node_guard = Some(allocated_guard); - } else if cur_entry.is_node() { - let ChildRef::PageTable(pt) = cur_entry.to_ref() else { - unreachable!(); - }; - cur_pt_addr = pt.paddr(); - cur_node_guard = None; - } else { - break; + match cur_entry.to_ref() { + ChildRef::Frame(_, _, _) => { + break; + } + ChildRef::None => { + let allocated_guard = cur_entry.alloc_if_none(guard).unwrap(); + cur_pt_addr = allocated_guard.paddr(); + cur_node_guard = Some(allocated_guard); + } + ChildRef::PageTable(pt) => { + cur_pt_addr = pt.paddr(); + cur_node_guard = None; + } } } diff --git a/ostd/src/mm/page_table/mod.rs b/ostd/src/mm/page_table/mod.rs index 90ad7083a..8d507e87b 100644 --- a/ostd/src/mm/page_table/mod.rs +++ b/ostd/src/mm/page_table/mod.rs @@ -19,6 +19,7 @@ use super::{ use crate::{ Pod, arch::mm::{PageTableEntry, PagingConsts}, + mm::page_prop::PageTableFlags, task::{atomic_mode::AsAtomicModeGuard, disable_preempt}, }; @@ -79,7 +80,7 @@ pub(crate) unsafe trait PageTableConfig: const TOP_LEVEL_CAN_UNMAP: bool = true; /// The type of the page table entry. - type E: PageTableEntryTrait; + type E: PteTrait; /// The paging constants. type C: PagingConstsTrait; @@ -342,7 +343,10 @@ impl PageTable { // outlive the kernel page table, which is trivially true. // See also `::on_drop`. let pt_addr = pt.paddr(); - let pte = PageTableEntry::new_pt(pt_addr); + let pte = PageTableEntry::from_repr( + &PteScalar::PageTable(pt_addr, PageTableFlags::empty()), + UserPtConfig::NR_LEVELS, + ); // SAFETY: The index is within the bounds and the PTE is at the // correct paging level. However, neither it's a `UserPtConfig` // child nor the node has the ownership of the child. It is @@ -467,71 +471,66 @@ pub(super) unsafe fn page_walk( // - All page table entries are aligned and accessed with atomic operations only. let cur_pte = unsafe { load_pte((pt_addr as *mut C::E).add(offset), Ordering::Acquire) }; - if !cur_pte.is_present() { - return None; + match cur_pte.to_repr(cur_level) { + PteScalar::Absent => return None, + PteScalar::PageTable(next_pt_addr, _) => { + pt_addr = paddr_to_vaddr(next_pt_addr); + continue; + } + PteScalar::Mapped(frame_paddr, prop) => { + debug_assert!(cur_level <= C::HIGHEST_TRANSLATION_LEVEL); + return Some(( + frame_paddr + (vaddr & (page_size::(cur_level) - 1)), + prop, + )); + } } - - if cur_pte.is_last(cur_level) { - debug_assert!(cur_level <= C::HIGHEST_TRANSLATION_LEVEL); - return Some(( - cur_pte.paddr() + (vaddr & (page_size::(cur_level) - 1)), - cur_pte.prop(), - )); - } - - pt_addr = paddr_to_vaddr(cur_pte.paddr()); } unreachable!("All present PTEs at the level 1 must be last-level PTEs"); } +/// The scalar representation of a page table entry (PTE). +/// +/// This is an architecture-agnostic representation that can be converted +/// to/from architecture-specific PTEs via the [`PteTrait`]. This is a scalar +/// value that can be cloned or compared, and does not own the underlying page +/// table node or mapped item if present. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum PteScalar { + /// A PTE that is considered absent by the MMU. + Absent, + /// A PTE that points to the next-level page table. + PageTable(Paddr, PageTableFlags), + /// A PTE that establishes the mapping to a physical frame. + Mapped(Paddr, PageProperty), +} + /// A trait that abstracts architecture-specific page table entries (PTEs). /// -/// Note that a default PTE should be a PTE that points to nothing. -pub trait PageTableEntryTrait: - Clone + Copy + Debug + Default + Pod + PodOnce + Sized + Send + Sync + 'static +/// A PTE refers to an entry in any level of a page table. This trait requires +/// that any architecture-specific PTEs are scalar types that essentially +/// encodes [`PteScalar`]. +/// +/// # Safety +/// +/// An implementor must ensure that: +/// - the methods `as_usize` and `from_usize` are not overridden; +/// - the return value of `from_repr`, when called with `repr`, should return +/// the same [`PteScalar`] that is passed to `from_repr`, if the level +/// passed to `repr` is the same as that passed to `from_repr`; +/// - a zeroed PTE represents an absent entry. +pub(crate) unsafe trait PteTrait: + Clone + Copy + Debug + Pod + PodOnce + Sized + Send + Sync + 'static { - /// Creates a PTE that points to nothing. + /// Returns architecture-specific representation of the PTE. + fn from_repr(repr: &PteScalar, level: PagingLevel) -> Self; + + /// Returns the representation of the PTE. /// - /// Note that currently the implementation requires a zeroed PTE to be an absent PTE. - fn new_absent() -> Self { - Self::default() - } - - /// Returns if the PTE points to something. - /// - /// For PTEs created by [`Self::new_absent`], this method should return - /// false. For PTEs created by [`Self::new_page`] or [`Self::new_pt`] - /// and modified with [`Self::set_prop`], this method should return true. - fn is_present(&self) -> bool; - - /// Creates a new PTE that maps to a page. - fn new_page(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self; - - /// Creates a new PTE that maps to a child page table. - fn new_pt(paddr: Paddr) -> Self; - - /// Returns the physical address from the PTE. - /// - /// The physical address recorded in the PTE is either: - /// - the physical address of the next-level page table, or - /// - the physical address of the page that the PTE maps to. - fn paddr(&self) -> Paddr; - - /// Returns the page property of the PTE. - fn prop(&self) -> PageProperty; - - /// Sets the page property of the PTE. - /// - /// This methold has an impact only if the PTE is present. If not, this - /// method will do nothing. - fn set_prop(&mut self, prop: PageProperty); - - /// Returns if the PTE maps a page rather than a child page table. - /// - /// The method needs to know the level of the page table where the PTE resides, - /// since architectures like x86-64 have a huge bit only in intermediate levels. - fn is_last(&self, level: PagingLevel) -> bool; + /// The caller must ensure that the level is the correct level of the PTE, + /// otherwise the implementation can return arbitrary value. + fn to_repr(&self, level: PagingLevel) -> PteScalar; /// Converts the PTE into a raw `usize` value. fn as_usize(self) -> usize { @@ -555,7 +554,7 @@ pub trait PageTableEntryTrait: /// # Safety /// /// The safety preconditions are same as those of [`AtomicUsize::from_ptr`]. -pub unsafe fn load_pte(ptr: *mut E, ordering: Ordering) -> E { +pub unsafe fn load_pte(ptr: *mut E, ordering: Ordering) -> E { // SAFETY: The safety is upheld by the caller. let atomic = unsafe { AtomicUsize::from_ptr(ptr.cast()) }; let pte_raw = atomic.load(ordering); @@ -567,7 +566,7 @@ pub unsafe fn load_pte(ptr: *mut E, ordering: Ordering) /// # Safety /// /// The safety preconditions are same as those of [`AtomicUsize::from_ptr`]. -pub unsafe fn store_pte(ptr: *mut E, new_val: E, ordering: Ordering) { +pub unsafe fn store_pte(ptr: *mut E, new_val: E, ordering: Ordering) { let new_raw = new_val.as_usize(); // SAFETY: The safety is upheld by the caller. let atomic = unsafe { AtomicUsize::from_ptr(ptr.cast()) }; diff --git a/ostd/src/mm/page_table/node/child.rs b/ostd/src/mm/page_table/node/child.rs index 1ebeba9ef..9eabc1a3f 100644 --- a/ostd/src/mm/page_table/node/child.rs +++ b/ostd/src/mm/page_table/node/child.rs @@ -4,9 +4,15 @@ use core::mem::ManuallyDrop; -use super::{PageTableEntryTrait, PageTableNode, PageTableNodeRef}; +use ostd_pod::Pod; + +use super::{PageTableNode, PageTableNodeRef, PteTrait}; use crate::{ - mm::{HasPaddr, Paddr, PagingLevel, page_prop::PageProperty, page_table::PageTableConfig}, + mm::{ + HasPaddr, Paddr, PageTableFlags, PagingLevel, + page_prop::PageProperty, + page_table::{PageTableConfig, PteScalar}, + }, sync::RcuDrop, }; @@ -33,11 +39,14 @@ impl Child { match self { Child::PageTable(node) => { let paddr = node.paddr(); + let level = node.level(); let _ = ManuallyDrop::new(node); - C::E::new_pt(paddr) + C::E::from_repr(&PteScalar::PageTable(paddr, PageTableFlags::empty()), level) } - Child::Frame(paddr, level, prop) => C::E::new_page(paddr, level, prop), - Child::None => C::E::new_absent(), + Child::Frame(paddr, level, prop) => { + C::E::from_repr(&PteScalar::Mapped(paddr, prop), level) + } + Child::None => C::E::new_zeroed(), } } @@ -49,21 +58,19 @@ impl Child { /// /// The level must match the original level of the child. pub(super) unsafe fn from_pte(pte: C::E, level: PagingLevel) -> Self { - if !pte.is_present() { - return Child::None; + let repr = pte.to_repr(level); + + match repr { + PteScalar::Absent => Child::None, + PteScalar::PageTable(paddr, _) => { + // SAFETY: The caller ensures that this node was created by + // `into_pte`, so that restoring the forgotten reference is safe. + let node = unsafe { PageTableNode::from_raw(paddr) }; + debug_assert_eq!(node.level(), level - 1); + Child::PageTable(RcuDrop::new(node)) + } + PteScalar::Mapped(paddr, prop) => Child::Frame(paddr, level, prop), } - - let paddr = pte.paddr(); - - if !pte.is_last(level) { - // SAFETY: The caller ensures that this node was created by - // `into_pte`, so that restoring the forgotten reference is safe. - let node = unsafe { PageTableNode::from_raw(paddr) }; - debug_assert_eq!(node.level(), level - 1); - return Child::PageTable(RcuDrop::new(node)); - } - - Child::Frame(paddr, level, pte.prop()) } } @@ -91,21 +98,19 @@ impl ChildRef<'_, C> { /// The provided level must be the same with the level of the page table /// node that contains this PTE. pub(super) unsafe fn from_pte(pte: &C::E, level: PagingLevel) -> Self { - if !pte.is_present() { - return ChildRef::None; + let repr = pte.to_repr(level); + + match repr { + PteScalar::Absent => ChildRef::None, + PteScalar::PageTable(paddr, _) => { + // SAFETY: The caller ensures that the lifetime of the child is + // contained by the residing node, and the physical address is + // valid since the entry is present. + let node = unsafe { PageTableNodeRef::borrow_paddr(paddr) }; + debug_assert_eq!(node.level(), level - 1); + ChildRef::PageTable(node) + } + PteScalar::Mapped(paddr, prop) => ChildRef::Frame(paddr, level, prop), } - - let paddr = pte.paddr(); - - if !pte.is_last(level) { - // SAFETY: The caller ensures that the lifetime of the child is - // contained by the residing node, and the physical address is - // valid since the entry is present. - let node = unsafe { PageTableNodeRef::borrow_paddr(paddr) }; - debug_assert_eq!(node.level(), level - 1); - return ChildRef::PageTable(node); - } - - ChildRef::Frame(paddr, level, pte.prop()) } } diff --git a/ostd/src/mm/page_table/node/entry.rs b/ostd/src/mm/page_table/node/entry.rs index 214d80bca..4a040ac42 100644 --- a/ostd/src/mm/page_table/node/entry.rs +++ b/ostd/src/mm/page_table/node/entry.rs @@ -2,13 +2,13 @@ //! This module provides accessors to the page table entries in a node. -use super::{Child, ChildRef, PageTableEntryTrait, PageTableGuard, PageTableNode}; +use super::{Child, ChildRef, PageTableGuard, PageTableNode, PteTrait}; use crate::{ mm::{ HasPaddr, nr_subpage_per_huge, page_prop::PageProperty, page_size, - page_table::{PageTableConfig, PageTableNodeRef}, + page_table::{PageTableConfig, PageTableNodeRef, PteScalar}, }, sync::RcuDrop, task::atomic_mode::InAtomicMode, @@ -37,16 +37,6 @@ pub(in crate::mm) struct Entry<'a, 'rcu, C: PageTableConfig> { } impl<'a, 'rcu, C: PageTableConfig> Entry<'a, 'rcu, C> { - /// Returns if the entry does not map to anything. - pub(in crate::mm) fn is_none(&self) -> bool { - !self.pte.is_present() - } - - /// Returns if the entry maps to a page table node. - pub(in crate::mm) fn is_node(&self) -> bool { - self.pte.is_present() && !self.pte.is_last(self.node.level()) - } - /// Gets a reference to the child. pub(in crate::mm) fn to_ref(&self) -> ChildRef<'rcu, C> { // SAFETY: @@ -59,11 +49,11 @@ impl<'a, 'rcu, C: PageTableConfig> Entry<'a, 'rcu, C> { /// /// It only modifies the properties if the entry is present. pub(in crate::mm) fn protect(&mut self, op: &mut impl FnMut(&mut PageProperty)) { - if !self.pte.is_present() { + let level = self.node.level(); + let PteScalar::Mapped(pa, prop) = self.pte.to_repr(level) else { return; - } + }; - let prop = self.pte.prop(); let mut new_prop = prop; op(&mut new_prop); @@ -71,7 +61,7 @@ impl<'a, 'rcu, C: PageTableConfig> Entry<'a, 'rcu, C> { return; } - self.pte.set_prop(new_prop); + self.pte = C::E::from_repr(&PteScalar::Mapped(pa, new_prop), level); // SAFETY: // 1. The index is within the bounds. @@ -130,7 +120,7 @@ impl<'a, 'rcu, C: PageTableConfig> Entry<'a, 'rcu, C> { &mut self, guard: &'rcu dyn InAtomicMode, ) -> Option> { - if !(self.is_none() && self.node.level() > 1) { + if !matches!(self.to_ref(), ChildRef::None) || self.node.level() == 1 { return None; } @@ -171,13 +161,9 @@ impl<'a, 'rcu, C: PageTableConfig> Entry<'a, 'rcu, C> { guard: &'rcu dyn InAtomicMode, ) -> Option> { let level = self.node.level(); - - if !(self.pte.is_last(level) && level > 1) { + let PteScalar::Mapped(pa, prop) = self.pte.to_repr(level) else { return None; - } - - let pa = self.pte.paddr(); - let prop = self.pte.prop(); + }; let new_page = RcuDrop::new(PageTableNode::::alloc(level - 1)); diff --git a/ostd/src/mm/page_table/node/mod.rs b/ostd/src/mm/page_table/node/mod.rs index 21cbafe39..e3a567cc8 100644 --- a/ostd/src/mm/page_table/node/mod.rs +++ b/ostd/src/mm/page_table/node/mod.rs @@ -39,13 +39,13 @@ pub(in crate::mm) use self::{ child::{Child, ChildRef}, entry::Entry, }; -use super::{PageTableConfig, PageTableEntryTrait, nr_subpage_per_huge}; +use super::{PageTableConfig, PteTrait, nr_subpage_per_huge}; use crate::{ mm::{ FrameAllocOptions, HasPaddr, Infallible, PagingConstsTrait, PagingLevel, VmReader, frame::{Frame, FrameRef, meta::AnyFrameMeta}, paddr_to_vaddr, - page_table::{load_pte, store_pte}, + page_table::{PteScalar, load_pte, store_pte}, }, task::atomic_mode::InAtomicMode, }; @@ -69,14 +69,10 @@ impl PageTableNode { /// Allocates a new empty page table node. pub(super) fn alloc(level: PagingLevel) -> Self { let meta = PageTablePageMeta::new(level); - let frame = FrameAllocOptions::new() + FrameAllocOptions::new() .zeroed(true) .alloc_frame_with(meta) - .expect("Failed to allocate a page table node"); - // The allocated frame is zeroed. Make sure zero is absent PTE. - debug_assert_eq!(C::E::new_absent().as_usize(), 0); - - frame + .expect("Failed to allocate a page table node") } /// Activates the page table assuming it is a root page table. @@ -318,20 +314,18 @@ unsafe impl AnyFrameMeta for PageTablePageMeta { for _ in range { // Non-atomic read is OK because we have mutable access. let pte = reader.read_once::().unwrap(); - if pte.is_present() { - let paddr = pte.paddr(); - // As a fast path, we can ensure that the type of the child frame - // is `Self` if the PTE points to a child page table. Then we don't - // need to check the vtable for the drop method. - if !pte.is_last(level) { + match pte.to_repr(level) { + PteScalar::PageTable(child_pt_addr, _) => { // SAFETY: The PTE points to a page table node. The ownership // of the child is transferred to the child then dropped. - drop(unsafe { Frame::::from_raw(paddr) }); - } else { + drop(unsafe { PageTableNode::::from_raw(child_pt_addr) }); + } + PteScalar::Mapped(pa, prop) => { // SAFETY: The PTE points to a mapped item. The ownership // of the item is transferred here then dropped. - drop(unsafe { C::item_from_raw(paddr, level, pte.prop()) }); + drop(unsafe { C::item_from_raw(pa, level, prop) }); } + PteScalar::Absent => {} } } } diff --git a/ostd/src/mm/page_table/test.rs b/ostd/src/mm/page_table/test.rs index e7b8c8b18..cdee20e59 100644 --- a/ostd/src/mm/page_table/test.rs +++ b/ostd/src/mm/page_table/test.rs @@ -14,6 +14,7 @@ use crate::{ mod test_utils { use super::*; + use crate::mm::PrivilegedPageFlags; /// Creates a new user page table that has mapped a virtual range to a physical frame. #[track_caller] @@ -111,6 +112,78 @@ mod test_utils { (paddr, level, prop) } } + + /// A subset iterator for bitflags. + /// + /// A bitflag is a set of boolean options represented as bits in an integer. + /// + /// When given a bitflag `full`, it iterates over all subsets of `full` in + /// descending order of their integer values. + pub struct SubsetIter { + full: u8, + cur: u8, + finished: bool, + } + + impl SubsetIter { + /// Create a new subset iterator for the given full bitflag. + pub fn new(full: u8) -> Self { + SubsetIter { + full, + cur: full, + finished: false, + } + } + } + + impl Iterator for SubsetIter { + type Item = u8; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + let flag = self.cur; + if self.cur == 0 { + self.finished = true; + } else { + self.cur = (self.cur - 1) & self.full; + } + Some(flag) + } + } + + #[ktest] + fn test_subset_iter() { + use alloc::{vec, vec::Vec}; + + assert_eq!( + SubsetIter::new(0b1011).collect::>(), + vec![ + 0b1011, 0b1010, 0b1001, 0b1000, 0b0011, 0b0010, 0b0001, 0b0000 + ] + ); + } + + /// Generates all possible page properties. + pub fn all_page_properties() -> impl Iterator { + let flag_subsets = + SubsetIter::new(PageFlags::all().bits()).map(|f| PageFlags::from_bits(f).unwrap()); + flag_subsets.flat_map(|flags| { + let priv_flag_subsets = SubsetIter::new(PrivilegedPageFlags::all().bits()) + .map(|f| PrivilegedPageFlags::from_bits(f).unwrap()); + priv_flag_subsets.flat_map(move |priv_flags| { + // We do not supporting other cache policies yet. So just test them. + static CACHE_POLICIES: [CachePolicy; 2] = + [CachePolicy::Writeback, CachePolicy::Uncacheable]; + CACHE_POLICIES.iter().map(move |&cache| PageProperty { + flags, + cache, + priv_flags, + }) + }) + }) + } } mod create_page_table { @@ -162,7 +235,10 @@ mod create_page_table { let preempt_guard = disable_preempt(); let mut root_node = kernel_pt.root.borrow().lock(&preempt_guard); for i in shared_range { - assert!(root_node.entry(i).is_node()); + assert!(matches!( + root_node.entry(i).to_ref(), + ChildRef::PageTable(_) + )); } } } @@ -245,7 +321,7 @@ mod range_checks { } mod page_properties { - use super::*; + use super::{test_utils::all_page_properties, *}; use crate::mm::PrivilegedPageFlags; /// Helper function to map a single page with given properties and verify the properties. @@ -273,45 +349,80 @@ mod page_properties { #[ktest] fn map_preserves_page_property() { - struct SubsetIter { - full: u8, - cur: u8, - } - impl SubsetIter { - fn new(full: u8) -> Self { - SubsetIter { full, cur: full } - } - } - impl Iterator for SubsetIter { - type Item = u8; - fn next(&mut self) -> Option { - if self.cur == 0 { - return None; - } - let flag = self.cur; - self.cur = (self.cur - 1) & self.full; - Some(flag) - } + for prop in all_page_properties() { + check_map_with_property(prop); } + } +} - let flag_subsets = - SubsetIter::new(PageFlags::all().bits()).map(|f| PageFlags::from_bits(f).unwrap()); - for flags in flag_subsets { - let priv_flag_subsets = SubsetIter::new(PrivilegedPageFlags::all().bits()) - .map(|f| PrivilegedPageFlags::from_bits(f).unwrap()); - for priv_flags in priv_flag_subsets { - // We do not supporting other cache policies yet. So just test them. - let cache_policies = [CachePolicy::Writeback, CachePolicy::Uncacheable]; - for cache in cache_policies { - check_map_with_property(PageProperty { - flags, - cache, - priv_flags, - }); +mod arch_pte_impls { + use super::{ + test_utils::{SubsetIter, all_page_properties}, + *, + }; + use crate::{ + arch::mm::{PageTableEntry, PagingConsts}, + mm::{ + PageTableFlags, + page_table::{PteScalar, PteTrait}, + }, + }; + + #[ktest] + fn zeroed_pte_is_absent_pte() { + let pte = PageTableEntry::new_zeroed(); + for level in 1..=PagingConsts::NR_LEVELS { + let repr = pte.to_repr(level); + assert_eq!(repr, PteScalar::Absent); + } + } + + #[ktest] + fn cast_frame_pte_preserves_repr() { + for level in 1..=PagingConsts::HIGHEST_TRANSLATION_LEVEL { + for prop in all_page_properties() { + // TODO: Almost all architectures doesn't support non-readable + // pages. We can opt-out this flag at compile time. + if !prop.flags.contains(PageFlags::R) { + continue; } + + let paddr = 0xff_c000_0000; + let repr = PteScalar::Mapped(paddr, prop); + let pte = PageTableEntry::from_repr(&repr, level); + let parsed_repr = pte.to_repr(level); + + assert_eq!(repr, parsed_repr); } } } + + #[ktest] + fn cast_pt_pte_preserves_repr() { + for level in 2..=PagingConsts::NR_LEVELS { + let paddr = 0xff_c000_0000; + let pt_flags_iter = SubsetIter::new(PageTableFlags::all().bits()) + .map(|f| PageTableFlags::from_bits(f).unwrap()); + for pt_flags in pt_flags_iter { + let repr = PteScalar::PageTable(paddr, pt_flags); + let pte = PageTableEntry::from_repr(&repr, level); + let parsed_repr = pte.to_repr(level); + + assert_eq!(repr, parsed_repr); + } + } + } + + #[ktest] + fn cast_absent_pte_preserves_repr() { + for level in 1..=PagingConsts::NR_LEVELS { + let repr = PteScalar::Absent; + let pte = PageTableEntry::from_repr(&repr, level); + let parsed_repr = pte.to_repr(level); + + assert_eq!(repr, parsed_repr); + } + } } mod overlapping_mappings { @@ -905,12 +1016,10 @@ mod boot_pt { fn map_base_page() { let root_frame = FrameAllocOptions::new().alloc_frame().unwrap(); let root_paddr = root_frame.paddr(); - let mut boot_pt = BootPageTable::::new( - root_paddr / PagingConsts::BASE_PAGE_SIZE, - ); + let mut boot_pt = BootPageTable::::new(root_paddr); let from_virt = 0x1000; - let to_phys = 0x2; + let to_phys = 0x2000; let page_property = PageProperty::new_user(PageFlags::RW, CachePolicy::Writeback); unsafe { @@ -921,7 +1030,7 @@ mod boot_pt { let root_paddr = boot_pt.root_address(); assert_eq!( unsafe { page_walk::(root_paddr, from_virt + 1) }, - Some((to_phys * PAGE_SIZE + 1, page_property)) + Some((to_phys + 1, page_property)) ); } @@ -930,13 +1039,11 @@ mod boot_pt { fn map_base_page_already_mapped() { let root_frame = FrameAllocOptions::new().alloc_frame().unwrap(); let root_paddr = root_frame.paddr(); - let mut boot_pt = BootPageTable::::new( - root_paddr / PagingConsts::BASE_PAGE_SIZE, - ); + let mut boot_pt = BootPageTable::::new(root_paddr); let from_virt = 0x1000; - let to_phys1 = 0x2; - let to_phys2 = 0x3; + let to_phys1 = 0x2000; + let to_phys2 = 0x3000; let page_property = PageProperty::new_user(PageFlags::RW, CachePolicy::Writeback); unsafe { @@ -950,9 +1057,7 @@ mod boot_pt { fn protect_base_page_unmapped() { let root_frame = FrameAllocOptions::new().alloc_frame().unwrap(); let root_paddr = root_frame.paddr(); - let mut boot_pt = BootPageTable::::new( - root_paddr / PagingConsts::BASE_PAGE_SIZE, - ); + let mut boot_pt = BootPageTable::::new(root_paddr); let virt_addr = 0x2000; // Attempts to protect an unmapped page (expected to panic). @@ -965,20 +1070,18 @@ mod boot_pt { fn map_protect() { let root_frame = FrameAllocOptions::new().alloc_frame().unwrap(); let root_paddr = root_frame.paddr(); - let mut boot_pt = BootPageTable::::new( - root_paddr / PagingConsts::BASE_PAGE_SIZE, - ); + let mut boot_pt = BootPageTable::::new(root_paddr); let root_paddr = boot_pt.root_address(); // Maps page 1. let from1 = 0x2000; - let to_phys1 = 0x2; + let to_phys1 = 0x2000; let prop1 = PageProperty::new_user(PageFlags::RW, CachePolicy::Writeback); unsafe { boot_pt.map_base_page(from1, to_phys1, prop1) }; assert_eq!( unsafe { page_walk::(root_paddr, from1 + 1) }, - Some((to_phys1 * PAGE_SIZE + 1, prop1)) + Some((to_phys1 + 1, prop1)) ); // Protects page 1. @@ -987,17 +1090,17 @@ mod boot_pt { PageProperty::new_user(PageFlags::RX, CachePolicy::Writeback); assert_eq!( unsafe { page_walk::(root_paddr, from1 + 1) }, - Some((to_phys1 * PAGE_SIZE + 1, expected_prop1_protected)) + Some((to_phys1 + 1, expected_prop1_protected)) ); // Maps page 2. let from2 = 0x3000; - let to_phys2 = 0x3; + let to_phys2 = 0x3000; let prop2 = PageProperty::new_user(PageFlags::RX, CachePolicy::Uncacheable); unsafe { boot_pt.map_base_page(from2, to_phys2, prop2) }; assert_eq!( unsafe { page_walk::(root_paddr, from2 + 2) }, - Some((to_phys2 * PAGE_SIZE + 2, prop2)) + Some((to_phys2 + 2, prop2)) ); // Protects page 2. @@ -1006,7 +1109,7 @@ mod boot_pt { PageProperty::new_user(PageFlags::RW, CachePolicy::Uncacheable); assert_eq!( unsafe { page_walk::(root_paddr, from2 + 2) }, - Some((to_phys2 * PAGE_SIZE + 2, expected_prop2_protected)) + Some((to_phys2 + 2, expected_prop2_protected)) ); } }