// SPDX-License-Identifier: GPL-2.0
//! Page table mapping operations for NVIDIA GPUs.
use core::marker::PhantomData;
use kernel::{
gpu::buddy::{
AllocatedBlocks,
GpuBuddyAllocFlags,
GpuBuddyAllocMode, //
},
io::io_write,
prelude::*,
ptr::Alignment,
rbtree::{
RBTree,
RBTreeNode, //
},
sizes::SZ_4K, //
};
use super::{
walk::{
PtWalkInner,
WalkPdeResult,
WalkResult, //
},
AperturePde,
AperturePte,
DualPdeOps,
MmuConfig,
MmuV2,
MmuV3,
MmuVersion,
PageTableLevel,
PdeOps,
PteOps, //
};
use crate::{
mm::{
GpuMm,
Pfn,
Vfn,
VramAddress,
PAGE_SIZE, //
},
num::{
IntoSafeCast, //
},
};
/// A pre-allocated and zeroed page table page.
///
/// Created during the mapping prepare phase and consumed during the execute phase.
/// Stored in an [`RBTree`] keyed by the PDE slot address (`install_addr`).
pub(in crate::mm) struct PreparedPtPage {
/// The allocated and zeroed page table page.
pub(in crate::mm) alloc: Pin<KBox<AllocatedBlocks>>,
/// Page table level -- needed to determine if this PT page is for a dual PDE.
pub(in crate::mm) level: PageTableLevel,
}
/// Page table mapper.
pub(in crate::mm) struct PtMapInner<M: MmuConfig> {
walker: PtWalkInner<M>,
pdb_addr: VramAddress,
_phantom: PhantomData<M>,
}
impl<M: MmuConfig> PtMapInner<M> {
/// Create a new [`PtMapInner`].
pub(super) fn new(pdb_addr: VramAddress) -> Self {
Self {
walker: PtWalkInner::<M>::new(pdb_addr),
pdb_addr,
_phantom: PhantomData,
}
}
/// Allocate and zero a physical page table page.
fn alloc_and_zero_page(mm: &mut GpuMm<'_>, level: PageTableLevel) -> Result<PreparedPtPage> {
let blocks = KBox::pin_init(
mm.buddy().alloc_blocks(
GpuBuddyAllocMode::Simple,
SZ_4K.into_safe_cast(),
Alignment::new::<SZ_4K>(),
GpuBuddyAllocFlags::default(),
),
GFP_KERNEL,
)?;
let page_vram = VramAddress::from_raw(blocks.iter().next().ok_or(ENOMEM)?.offset());
// Zero via PRAMIN.
let window = mm
.pramin_mut()
.window_at::<[u64; PAGE_SIZE / 8]>(page_vram)?;
for i in 0..PAGE_SIZE / 8 {
io_write!(window.view(), [build: i], 0);
}
Ok(PreparedPtPage {
alloc: blocks,
level,
})
}
/// Ensure all intermediate page table pages exist for a single VFN.
///
/// The mutable PRAMIN borrow ends before each allocation.
fn ensure_single_pte_path(
&self,
mm: &mut GpuMm<'_>,
vfn: Vfn,
pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
) -> Result {
let max_iter = 2 * M::PDE_LEVELS.len();
for _ in 0..max_iter {
let result = self
.walker
.walk_pde_levels(mm.pramin_mut(), vfn, |install_addr| {
pt_pages.get(&install_addr).and_then(|p| {
p.alloc
.iter()
.next()
.map(|b| VramAddress::from_raw(b.offset()))
})
})?;
match result {
WalkPdeResult::Complete { .. } => {
return Ok(());
}
WalkPdeResult::Missing {
install_addr,
level,
} => {
let page = Self::alloc_and_zero_page(mm, level)?;
let node = RBTreeNode::new(install_addr, page, GFP_KERNEL)?;
let old = pt_pages.insert(node);
if old.is_some() {
kernel::pr_warn_once!(
"VMM: duplicate install_addr in pt_pages (internal consistency error)\n"
);
return Err(EIO);
}
}
}
}
kernel::pr_warn!(
"VMM: ensure_pte_path: loop exhausted after {} iters (VFN {:?})\n",
max_iter,
vfn
);
Err(EIO)
}
/// Prepare page table resources for mapping `num_pages` pages starting at `vfn_start`.
///
/// Reserves capacity in `page_table_allocs`, then walks the hierarchy
/// per-VFN to prepare pages for all missing PDEs.
pub(super) fn prepare_map(
&self,
mm: &mut GpuMm<'_>,
vfn_start: Vfn,
num_pages: usize,
page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
) -> Result {
// Pre-reserve so install_mappings() can use push_within_capacity (no alloc
// in fence signalling critical path).
let pt_upper_bound = M::pt_pages_upper_bound(num_pages);
page_table_allocs.reserve(pt_upper_bound, GFP_KERNEL)?;
// Walk the hierarchy per-VFN to prepare pages for all missing PDEs.
for i in 0..num_pages {
let i_u64: u64 = i.into_safe_cast();
let vfn = Vfn::new(vfn_start.raw() + i_u64);
self.ensure_single_pte_path(mm, vfn, pt_pages)?;
}
Ok(())
}
/// Install prepared PDEs and write PTEs, then flush TLB.
///
/// Drains `pt_pages` and moves allocations into `page_table_allocs`.
pub(super) fn install_mappings(
&self,
mm: &mut GpuMm<'_>,
pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
vfn_start: Vfn,
pfns: &[Pfn],
writable: bool,
) -> Result {
{
let pramin = mm.pramin_mut();
// Drain prepared PT pages, install all pending PDEs.
let mut cursor = pt_pages.cursor_front_mut();
while let Some(c) = cursor {
let (next, node) = c.remove_current();
let (install_addr, page) = node.to_key_value();
let page_vram =
VramAddress::from_raw(page.alloc.iter().next().ok_or(ENOMEM)?.offset());
if page.level == M::DUAL_PDE_LEVEL {
let new_dpde = M::DualPde::new_small(Pfn::from(page_vram));
new_dpde.write(pramin, install_addr)?;
} else {
let new_pde = M::Pde::new(AperturePde::VideoMemory, Pfn::from(page_vram));
new_pde.write(pramin, install_addr)?;
}
page_table_allocs
.push_within_capacity(page.alloc)
.map_err(|_| ENOMEM)?;
cursor = next;
}
// Write PTEs (all PDEs now installed in HW).
for (i, &pfn) in pfns.iter().enumerate() {
let i_u64: u64 = i.into_safe_cast();
let vfn = Vfn::new(vfn_start.raw() + i_u64);
let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?;
match result {
WalkResult::Unmapped { pte_addr } | WalkResult::Mapped { pte_addr, .. } => {
let pte = M::Pte::new(AperturePte::VideoMemory, pfn, writable);
pte.write(pramin, pte_addr)?;
}
WalkResult::PageTableMissing => {
kernel::pr_warn_once!("VMM: page table missing for VFN {vfn:?}\n");
return Err(EIO);
}
}
}
}
// Flush TLB.
mm.tlb().flush(self.pdb_addr)
}
/// Invalidate PTEs for a range and flush TLB.
pub(super) fn invalidate_ptes(
&self,
mm: &mut GpuMm<'_>,
vfn_start: Vfn,
num_pages: usize,
) -> Result {
let invalid_pte = M::Pte::invalid();
{
let pramin = mm.pramin_mut();
for i in 0..num_pages {
let i_u64: u64 = i.into_safe_cast();
let vfn = Vfn::new(vfn_start.raw() + i_u64);
let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?;
match result {
WalkResult::Mapped { pte_addr, .. } | WalkResult::Unmapped { pte_addr } => {
invalid_pte.write(pramin, pte_addr)?;
}
WalkResult::PageTableMissing => {
continue;
}
}
}
}
mm.tlb().flush(self.pdb_addr)
}
}
macro_rules! pt_map_dispatch {
($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => {
match $self {
PtMap::V2(inner) => inner.$method($($arg),*),
PtMap::V3(inner) => inner.$method($($arg),*),
}
};
}
/// Page table mapper dispatch.
pub(in crate::mm) enum PtMap {
/// MMU v2 (Turing/Ampere/Ada).
V2(PtMapInner<MmuV2>),
/// MMU v3 (Hopper+).
V3(PtMapInner<MmuV3>),
}
impl PtMap {
/// Create a new page table mapper for the given MMU version.
pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self {
match version {
MmuVersion::V2 => Self::V2(PtMapInner::<MmuV2>::new(pdb_addr)),
MmuVersion::V3 => Self::V3(PtMapInner::<MmuV3>::new(pdb_addr)),
}
}
/// Prepare page table resources for a mapping.
pub(in crate::mm) fn prepare_map(
&self,
mm: &mut GpuMm<'_>,
vfn_start: Vfn,
num_pages: usize,
page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
) -> Result {
pt_map_dispatch!(
self,
prepare_map(mm, vfn_start, num_pages, page_table_allocs, pt_pages)
)
}
/// Install prepared PDEs and write PTEs, then flush TLB.
pub(in crate::mm) fn install_mappings(
&self,
mm: &mut GpuMm<'_>,
pt_pages: &mut RBTree<VramAddress, PreparedPtPage>,
page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>,
vfn_start: Vfn,
pfns: &[Pfn],
writable: bool,
) -> Result {
pt_map_dispatch!(
self,
install_mappings(mm, pt_pages, page_table_allocs, vfn_start, pfns, writable)
)
}
/// Invalidate PTEs for a range and flush TLB.
pub(in crate::mm) fn invalidate_ptes(
&self,
mm: &mut GpuMm<'_>,
vfn_start: Vfn,
num_pages: usize,
) -> Result {
pt_map_dispatch!(self, invalidate_ptes(mm, vfn_start, num_pages))
}
}