1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 pub(crate) mod commands; 5 mod r570_144; 6 7 // Alias to avoid repeating the version number with every use. 8 use r570_144 as bindings; 9 10 use core::ops::Range; 11 12 use kernel::{ 13 dma::Coherent, 14 prelude::*, 15 ptr::{ 16 Alignable, 17 Alignment, 18 KnownSize, // 19 }, 20 sizes::{ 21 SizeConstants, 22 SZ_128K, // 23 }, 24 transmute::{ 25 AsBytes, 26 FromBytes, // 27 }, 28 }; 29 30 use crate::{ 31 fb::FbLayout, 32 firmware::gsp::GspFirmware, 33 gpu::{ 34 Architecture, 35 Chipset, // 36 }, 37 gsp::{ 38 cmdq::Cmdq, // 39 GSP_PAGE_SIZE, 40 }, 41 num::{ 42 self, 43 FromSafeCast, // 44 }, 45 }; 46 47 // TODO: Replace with `IoView` projections once available. 48 pub(super) mod gsp_mem { 49 use core::sync::atomic::{ 50 fence, 51 Ordering, // 52 }; 53 54 use kernel::{ 55 dma::Coherent, 56 dma_read, 57 dma_write, // 58 }; 59 60 use crate::gsp::cmdq::{ 61 GspMem, 62 MSGQ_NUM_PAGES, // 63 }; 64 65 pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 { 66 dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES 67 } 68 69 pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 { 70 dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES 71 } 72 73 pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 { 74 dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES 75 } 76 77 pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) { 78 let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 79 80 // Ensure read pointer is properly ordered. 81 fence(Ordering::SeqCst); 82 83 dma_write!(qs, .cpuq.rx.0.readPtr, rptr); 84 } 85 86 pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 { 87 dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES 88 } 89 90 pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) { 91 let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 92 93 dma_write!(qs, .cpuq.tx.0.writePtr, wptr); 94 95 // Ensure all command data is visible before triggering the GSP read. 96 fence(Ordering::SeqCst); 97 } 98 } 99 100 /// Maximum size of a single GSP message queue element in bytes. 101 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 102 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 103 104 /// Empty type to group methods related to heap parameters for running the GSP firmware. 105 enum GspFwHeapParams {} 106 107 /// Minimum required alignment for the GSP heap. 108 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 109 110 impl GspFwHeapParams { 111 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 112 /// and including the first client subdevice allocation). 113 fn base_rm_size(chipset: Chipset) -> u64 { 114 match chipset.arch() { 115 Architecture::Turing | Architecture::Ampere | Architecture::Ada => { 116 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 117 } 118 Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { 119 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 120 } 121 } 122 } 123 124 /// Returns the amount of heap memory required to support a single channel allocation. 125 fn client_alloc_size() -> u64 { 126 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 127 .align_up(GSP_HEAP_ALIGNMENT) 128 .unwrap_or(u64::MAX) 129 } 130 131 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 132 /// `fb_size`. 133 fn management_overhead(fb_size: u64) -> Result<u64> { 134 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 135 136 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 137 .checked_mul(fb_size_gb) 138 .ok_or(EINVAL)? 139 .align_up(GSP_HEAP_ALIGNMENT) 140 .ok_or(EINVAL) 141 } 142 } 143 144 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 145 pub(crate) struct LibosParams { 146 /// The base amount of heap required by the GSP operating system, in bytes. 147 carveout_size: u64, 148 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 149 allowed_heap_size: Range<u64>, 150 } 151 152 impl LibosParams { 153 /// Version 2 of the GSP LIBOS (Turing and GA100) 154 const LIBOS2: LibosParams = LibosParams { 155 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 156 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 157 * u64::SZ_1M 158 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 159 }; 160 161 /// Version 3 of the GSP LIBOS (GA102+) 162 const LIBOS3: LibosParams = LibosParams { 163 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 164 allowed_heap_size: num::u32_as_u64( 165 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 166 ) * u64::SZ_1M 167 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 168 * u64::SZ_1M, 169 }; 170 171 /// Returns the libos parameters corresponding to `chipset`. 172 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 173 if chipset < Chipset::GA102 { 174 &Self::LIBOS2 175 } else { 176 &Self::LIBOS3 177 } 178 } 179 180 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 181 /// of `fb_size` (in bytes) for `chipset`. 182 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 183 // The WPR heap will contain the following: 184 // LIBOS carveout, 185 Ok(self 186 .carveout_size 187 // RM boot working memory, 188 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 189 // One RM client, 190 .saturating_add(GspFwHeapParams::client_alloc_size()) 191 // Overhead for memory management. 192 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 193 // Clamp to the supported heap sizes. 194 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 195 } 196 } 197 198 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 199 /// addresses of the GSP bootloader and firmware. 200 #[repr(transparent)] 201 pub(crate) struct GspFwWprMeta { 202 inner: bindings::GspFwWprMeta, 203 } 204 205 // SAFETY: Padding is explicit and does not contain uninitialized data. 206 unsafe impl AsBytes for GspFwWprMeta {} 207 208 // SAFETY: This struct only contains integer types for which all bit patterns 209 // are valid. 210 unsafe impl FromBytes for GspFwWprMeta {} 211 212 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 213 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 214 215 impl GspFwWprMeta { 216 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 217 /// `fb_layout` layout. 218 pub(crate) fn new<'a>( 219 gsp_firmware: &'a GspFirmware, 220 fb_layout: &'a FbLayout, 221 ) -> impl Init<Self> + 'a { 222 #[allow(non_snake_case)] 223 let init_inner = init!(bindings::GspFwWprMeta { 224 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 225 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 226 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 227 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), 228 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 229 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), 230 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 231 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 232 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 233 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 234 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 235 __bindgen_anon_1: GspFwWprMetaBootInfo { 236 sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), 237 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 238 }, 239 }, 240 gspFwRsvdStart: fb_layout.heap.start, 241 nonWprHeapOffset: fb_layout.heap.start, 242 nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, 243 gspFwWprStart: fb_layout.wpr2.start, 244 gspFwHeapOffset: fb_layout.wpr2_heap.start, 245 gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, 246 gspFwOffset: fb_layout.elf.start, 247 bootBinOffset: fb_layout.boot.start, 248 frtsOffset: fb_layout.frts.start, 249 frtsSize: fb_layout.frts.end - fb_layout.frts.start, 250 gspFwWprEnd: fb_layout 251 .vga_workspace 252 .start 253 .align_down(Alignment::new::<SZ_128K>()), 254 gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, 255 fbSize: fb_layout.fb.end - fb_layout.fb.start, 256 vgaWorkspaceOffset: fb_layout.vga_workspace.start, 257 vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, 258 pmuReservedSize: fb_layout.pmu_reserved_size, 259 ..Zeroable::init_zeroed() 260 }); 261 262 init!(GspFwWprMeta { 263 inner <- init_inner, 264 }) 265 } 266 } 267 268 #[derive(Copy, Clone, Debug, PartialEq)] 269 #[repr(u32)] 270 pub(crate) enum MsgFunction { 271 // Common function codes 272 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 273 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 274 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 275 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 276 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 277 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 278 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 279 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 280 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 281 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 282 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 283 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 284 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 285 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 286 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 287 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 288 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 289 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 290 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 291 UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, 292 293 // Event codes 294 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 295 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 296 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 297 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 298 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 299 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 300 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 301 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 302 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 303 } 304 305 impl TryFrom<u32> for MsgFunction { 306 type Error = kernel::error::Error; 307 308 fn try_from(value: u32) -> Result<MsgFunction> { 309 match value { 310 // Common function codes 311 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 312 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 313 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 314 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 315 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 316 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 317 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 318 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 319 Ok(MsgFunction::ContinuationRecord) 320 } 321 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 322 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 323 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 324 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 325 Ok(MsgFunction::GspInitPostObjGpu) 326 } 327 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 328 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 329 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 330 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 331 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 332 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 333 Ok(MsgFunction::SetGuestSystemInfo) 334 } 335 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 336 bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { 337 Ok(MsgFunction::UnloadingGuestDriver) 338 } 339 340 // Event codes 341 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 342 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 343 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 344 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 345 Ok(MsgFunction::GspRunCpuSequencer) 346 } 347 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 348 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 349 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 350 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 351 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 352 _ => Err(EINVAL), 353 } 354 } 355 } 356 357 impl From<MsgFunction> for u32 { 358 fn from(value: MsgFunction) -> Self { 359 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 360 value as u32 361 } 362 } 363 364 /// Sequencer buffer opcode for GSP sequencer commands. 365 #[derive(Copy, Clone, Debug, PartialEq)] 366 #[repr(u32)] 367 pub(crate) enum SeqBufOpcode { 368 // Core operation opcodes 369 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 370 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 371 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 372 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 373 374 // Delay opcode 375 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 376 377 // Register operation opcodes 378 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 379 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 380 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 381 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 382 } 383 384 impl TryFrom<u32> for SeqBufOpcode { 385 type Error = kernel::error::Error; 386 387 fn try_from(value: u32) -> Result<SeqBufOpcode> { 388 match value { 389 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 390 Ok(SeqBufOpcode::CoreReset) 391 } 392 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 393 Ok(SeqBufOpcode::CoreResume) 394 } 395 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 396 Ok(SeqBufOpcode::CoreStart) 397 } 398 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 399 Ok(SeqBufOpcode::CoreWaitForHalt) 400 } 401 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 402 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 403 Ok(SeqBufOpcode::RegModify) 404 } 405 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 406 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 407 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 408 _ => Err(EINVAL), 409 } 410 } 411 } 412 413 impl From<SeqBufOpcode> for u32 { 414 fn from(value: SeqBufOpcode) -> Self { 415 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 416 value as u32 417 } 418 } 419 420 /// Wrapper for GSP sequencer register write payload. 421 #[repr(transparent)] 422 #[derive(Copy, Clone, Debug)] 423 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 424 425 impl RegWritePayload { 426 /// Returns the register address. 427 pub(crate) fn addr(&self) -> u32 { 428 self.0.addr 429 } 430 431 /// Returns the value to write. 432 pub(crate) fn val(&self) -> u32 { 433 self.0.val 434 } 435 } 436 437 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 438 unsafe impl FromBytes for RegWritePayload {} 439 440 // SAFETY: Padding is explicit and will not contain uninitialized data. 441 unsafe impl AsBytes for RegWritePayload {} 442 443 /// Wrapper for GSP sequencer register modify payload. 444 #[repr(transparent)] 445 #[derive(Copy, Clone, Debug)] 446 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 447 448 impl RegModifyPayload { 449 /// Returns the register address. 450 pub(crate) fn addr(&self) -> u32 { 451 self.0.addr 452 } 453 454 /// Returns the mask to apply. 455 pub(crate) fn mask(&self) -> u32 { 456 self.0.mask 457 } 458 459 /// Returns the value to write. 460 pub(crate) fn val(&self) -> u32 { 461 self.0.val 462 } 463 } 464 465 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 466 unsafe impl FromBytes for RegModifyPayload {} 467 468 // SAFETY: Padding is explicit and will not contain uninitialized data. 469 unsafe impl AsBytes for RegModifyPayload {} 470 471 /// Wrapper for GSP sequencer register poll payload. 472 #[repr(transparent)] 473 #[derive(Copy, Clone, Debug)] 474 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 475 476 impl RegPollPayload { 477 /// Returns the register address. 478 pub(crate) fn addr(&self) -> u32 { 479 self.0.addr 480 } 481 482 /// Returns the mask to apply. 483 pub(crate) fn mask(&self) -> u32 { 484 self.0.mask 485 } 486 487 /// Returns the expected value. 488 pub(crate) fn val(&self) -> u32 { 489 self.0.val 490 } 491 492 /// Returns the timeout in microseconds. 493 pub(crate) fn timeout(&self) -> u32 { 494 self.0.timeout 495 } 496 } 497 498 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 499 unsafe impl FromBytes for RegPollPayload {} 500 501 // SAFETY: Padding is explicit and will not contain uninitialized data. 502 unsafe impl AsBytes for RegPollPayload {} 503 504 /// Wrapper for GSP sequencer delay payload. 505 #[repr(transparent)] 506 #[derive(Copy, Clone, Debug)] 507 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 508 509 impl DelayUsPayload { 510 /// Returns the delay value in microseconds. 511 pub(crate) fn val(&self) -> u32 { 512 self.0.val 513 } 514 } 515 516 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 517 unsafe impl FromBytes for DelayUsPayload {} 518 519 // SAFETY: Padding is explicit and will not contain uninitialized data. 520 unsafe impl AsBytes for DelayUsPayload {} 521 522 /// Wrapper for GSP sequencer register store payload. 523 #[repr(transparent)] 524 #[derive(Copy, Clone, Debug)] 525 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 526 527 impl RegStorePayload { 528 /// Returns the register address. 529 pub(crate) fn addr(&self) -> u32 { 530 self.0.addr 531 } 532 533 /// Returns the storage index. 534 #[allow(unused)] 535 pub(crate) fn index(&self) -> u32 { 536 self.0.index 537 } 538 } 539 540 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 541 unsafe impl FromBytes for RegStorePayload {} 542 543 // SAFETY: Padding is explicit and will not contain uninitialized data. 544 unsafe impl AsBytes for RegStorePayload {} 545 546 /// Wrapper for GSP sequencer buffer command. 547 #[repr(transparent)] 548 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 549 550 impl SequencerBufferCmd { 551 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 552 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 553 self.0.opCode.try_into() 554 } 555 556 /// Returns the register write payload by value. 557 /// 558 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 559 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 560 if self.opcode()? != SeqBufOpcode::RegWrite { 561 return Err(EINVAL); 562 } 563 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 564 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 565 } 566 567 /// Returns the register modify payload by value. 568 /// 569 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 570 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 571 if self.opcode()? != SeqBufOpcode::RegModify { 572 return Err(EINVAL); 573 } 574 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 575 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 576 } 577 578 /// Returns the register poll payload by value. 579 /// 580 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 581 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 582 if self.opcode()? != SeqBufOpcode::RegPoll { 583 return Err(EINVAL); 584 } 585 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 586 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 587 } 588 589 /// Returns the delay payload by value. 590 /// 591 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 592 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 593 if self.opcode()? != SeqBufOpcode::DelayUs { 594 return Err(EINVAL); 595 } 596 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 597 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 598 } 599 600 /// Returns the register store payload by value. 601 /// 602 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 603 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 604 if self.opcode()? != SeqBufOpcode::RegStore { 605 return Err(EINVAL); 606 } 607 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 608 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 609 } 610 } 611 612 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 613 unsafe impl FromBytes for SequencerBufferCmd {} 614 615 // SAFETY: Padding is explicit and will not contain uninitialized data. 616 unsafe impl AsBytes for SequencerBufferCmd {} 617 618 /// Wrapper for GSP run CPU sequencer RPC. 619 #[repr(transparent)] 620 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 621 622 impl RunCpuSequencer { 623 /// Returns the command index. 624 pub(crate) fn cmd_index(&self) -> u32 { 625 self.0.cmdIndex 626 } 627 } 628 629 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 630 unsafe impl FromBytes for RunCpuSequencer {} 631 632 // SAFETY: Padding is explicit and will not contain uninitialized data. 633 unsafe impl AsBytes for RunCpuSequencer {} 634 635 /// Struct containing the arguments required to pass a memory buffer to the GSP 636 /// for use during initialisation. 637 /// 638 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 639 /// configured for a larger page size (e.g. 64K pages), we need to give 640 /// the GSP an array of 4K pages. Since we only create physically contiguous 641 /// buffers the math to calculate the addresses is simple. 642 /// 643 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 644 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 645 /// buffers to be physically contiguous anyway. 646 /// 647 /// The memory allocated for the arguments must remain until the GSP sends the 648 /// init_done RPC. 649 #[repr(transparent)] 650 pub(crate) struct LibosMemoryRegionInitArgument { 651 inner: bindings::LibosMemoryRegionInitArgument, 652 } 653 654 // SAFETY: Padding is explicit and does not contain uninitialized data. 655 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 656 657 // SAFETY: This struct only contains integer types for which all bit patterns 658 // are valid. 659 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 660 661 impl LibosMemoryRegionInitArgument { 662 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 663 name: &'static str, 664 obj: &'a Coherent<A>, 665 ) -> impl Init<Self> + 'a { 666 /// Generates the `ID8` identifier required for some GSP objects. 667 fn id8(name: &str) -> u64 { 668 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 669 670 for (c, b) in name.bytes().rev().zip(&mut bytes) { 671 *b = c; 672 } 673 674 u64::from_ne_bytes(bytes) 675 } 676 677 #[allow(non_snake_case)] 678 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 679 id8: id8(name), 680 pa: obj.dma_handle(), 681 size: num::usize_as_u64(obj.size()), 682 kind: num::u32_into_u8::< 683 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 684 >(), 685 loc: num::u32_into_u8::< 686 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 687 >(), 688 ..Zeroable::init_zeroed() 689 }); 690 691 init!(LibosMemoryRegionInitArgument { 692 inner <- init_inner, 693 }) 694 } 695 } 696 697 /// TX header for setting up a message queue with the GSP. 698 #[repr(transparent)] 699 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 700 701 impl MsgqTxHeader { 702 /// Create a new TX queue header. 703 /// 704 /// # Arguments 705 /// 706 /// * `msgq_size` - Total size of the message queue structure, in bytes. 707 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 708 /// structure. 709 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 710 /// allocated for the message queue in the message queue structure. 711 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 712 Self(bindings::msgqTxHeader { 713 version: 0, 714 size: msgq_size, 715 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 716 msgCount: msg_count, 717 writePtr: 0, 718 flags: 1, 719 rxHdrOff: rx_hdr_offset, 720 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 721 }) 722 } 723 } 724 725 // SAFETY: Padding is explicit and does not contain uninitialized data. 726 unsafe impl AsBytes for MsgqTxHeader {} 727 728 /// RX header for setting up a message queue with the GSP. 729 #[repr(transparent)] 730 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 731 732 /// Header for the message RX queue. 733 impl MsgqRxHeader { 734 /// Creates a new RX queue header. 735 pub(crate) fn new() -> Self { 736 Self(Default::default()) 737 } 738 } 739 740 // SAFETY: Padding is explicit and does not contain uninitialized data. 741 unsafe impl AsBytes for MsgqRxHeader {} 742 743 bitfield! { 744 struct MsgHeaderVersion(u32) { 745 31:24 major as u8; 746 23:16 minor as u8; 747 } 748 } 749 750 impl MsgHeaderVersion { 751 const MAJOR_TOT: u8 = 3; 752 const MINOR_TOT: u8 = 0; 753 754 fn new() -> Self { 755 Self::default() 756 .set_major(Self::MAJOR_TOT) 757 .set_minor(Self::MINOR_TOT) 758 } 759 } 760 761 impl bindings::rpc_message_header_v { 762 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 763 type RpcMessageHeader = bindings::rpc_message_header_v; 764 765 try_init!(RpcMessageHeader { 766 header_version: MsgHeaderVersion::new().into(), 767 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 768 function: function.into(), 769 length: size_of::<Self>() 770 .checked_add(cmd_size) 771 .ok_or(EOVERFLOW) 772 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 773 rpc_result: 0xffffffff, 774 rpc_result_private: 0xffffffff, 775 ..Zeroable::init_zeroed() 776 }) 777 } 778 } 779 780 /// GSP Message Element. 781 /// 782 /// This is essentially a message header expected to be followed by the message data. 783 #[repr(transparent)] 784 pub(crate) struct GspMsgElement { 785 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 786 } 787 788 impl GspMsgElement { 789 /// Creates a new message element. 790 /// 791 /// # Arguments 792 /// 793 /// * `sequence` - Sequence number of the message. 794 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 795 /// * `function` - Function of the message. 796 #[allow(non_snake_case)] 797 pub(crate) fn init( 798 sequence: u32, 799 cmd_size: usize, 800 function: MsgFunction, 801 ) -> impl Init<Self, Error> { 802 type RpcMessageHeader = bindings::rpc_message_header_v; 803 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 804 let init_inner = try_init!(InnerGspMsgElement { 805 seqNum: sequence, 806 elemCount: size_of::<Self>() 807 .checked_add(cmd_size) 808 .ok_or(EOVERFLOW)? 809 .div_ceil(GSP_PAGE_SIZE) 810 .try_into() 811 .map_err(|_| EOVERFLOW)?, 812 rpc <- RpcMessageHeader::init(cmd_size, function), 813 ..Zeroable::init_zeroed() 814 }); 815 816 try_init!(GspMsgElement { 817 inner <- init_inner, 818 }) 819 } 820 821 /// Sets the checksum of this message. 822 /// 823 /// Since the header is also part of the checksum, this is usually called after the whole 824 /// message has been written to the shared memory area. 825 pub(crate) fn set_checksum(&mut self, checksum: u32) { 826 self.inner.checkSum = checksum; 827 } 828 829 /// Returns the length of the message's payload. 830 pub(crate) fn payload_length(&self) -> usize { 831 // `rpc.length` includes the length of the RPC message header. 832 num::u32_as_usize(self.inner.rpc.length) 833 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 834 } 835 836 /// Returns the total length of the message, message and RPC headers included. 837 pub(crate) fn length(&self) -> usize { 838 size_of::<Self>() + self.payload_length() 839 } 840 841 // Returns the sequence number of the message. 842 pub(crate) fn sequence(&self) -> u32 { 843 self.inner.rpc.sequence 844 } 845 846 // Returns the function of the message, if it is valid, or the invalid function number as an 847 // error. 848 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 849 self.inner 850 .rpc 851 .function 852 .try_into() 853 .map_err(|_| self.inner.rpc.function) 854 } 855 856 // Returns the number of elements (i.e. memory pages) used by this message. 857 pub(crate) fn element_count(&self) -> u32 { 858 self.inner.elemCount 859 } 860 } 861 862 // SAFETY: Padding is explicit and does not contain uninitialized data. 863 unsafe impl AsBytes for GspMsgElement {} 864 865 // SAFETY: This struct only contains integer types for which all bit patterns 866 // are valid. 867 unsafe impl FromBytes for GspMsgElement {} 868 869 /// Arguments for GSP startup. 870 #[repr(transparent)] 871 #[derive(Zeroable)] 872 pub(crate) struct GspArgumentsCached { 873 inner: bindings::GSP_ARGUMENTS_CACHED, 874 } 875 876 impl GspArgumentsCached { 877 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 878 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 879 #[allow(non_snake_case)] 880 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 881 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 882 bDmemStack: 1, 883 ..Zeroable::init_zeroed() 884 }); 885 886 init!(GspArgumentsCached { 887 inner <- init_inner, 888 }) 889 } 890 } 891 892 // SAFETY: Padding is explicit and will not contain uninitialized data. 893 unsafe impl AsBytes for GspArgumentsCached {} 894 895 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 896 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 897 /// to that size. 898 #[repr(C)] 899 #[derive(Zeroable)] 900 pub(crate) struct GspArgumentsPadded { 901 pub(crate) inner: GspArgumentsCached, 902 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 903 } 904 905 impl GspArgumentsPadded { 906 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 907 init!(GspArgumentsPadded { 908 inner <- GspArgumentsCached::new(cmdq), 909 ..Zeroable::init_zeroed() 910 }) 911 } 912 } 913 914 // SAFETY: Padding is explicit and will not contain uninitialized data. 915 unsafe impl AsBytes for GspArgumentsPadded {} 916 917 // SAFETY: This struct only contains integer types for which all bit patterns 918 // are valid. 919 unsafe impl FromBytes for GspArgumentsPadded {} 920 921 /// Init arguments for the message queue. 922 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 923 924 impl MessageQueueInitArguments { 925 /// Creates a new init arguments structure for `cmdq`. 926 #[allow(non_snake_case)] 927 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 928 init!(MessageQueueInitArguments { 929 sharedMemPhysAddr: cmdq.dma_handle, 930 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 931 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 932 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 933 ..Zeroable::init_zeroed() 934 }) 935 } 936 } 937 938 #[repr(u32)] 939 pub(crate) enum GspDmaTarget { 940 #[expect(dead_code)] 941 LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, 942 CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, 943 NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, 944 } 945 946 type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; 947 948 impl GspAcrBootGspRmParams { 949 fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { 950 #[allow(non_snake_case)] 951 let params = init!(Self { 952 target: target as u32, 953 gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), 954 gspRmDescOffset: wpr_meta_addr, 955 bIsGspRmBoot: 1, 956 wprCarveoutOffset: 0, 957 wprCarveoutSize: 0, 958 __bindgen_padding_0: Default::default(), 959 }); 960 961 params 962 } 963 } 964 965 type GspRmParams = bindings::GSP_RM_PARAMS; 966 967 impl GspRmParams { 968 fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { 969 #[allow(non_snake_case)] 970 let params = init!(Self { 971 target: target as u32, 972 bootArgsOffset: libos_addr, 973 __bindgen_padding_0: Default::default(), 974 }); 975 976 params 977 } 978 } 979 980 pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; 981 982 // SAFETY: Padding is explicit and will not contain uninitialized data. 983 unsafe impl AsBytes for GspFmcBootParams {} 984 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 985 unsafe impl FromBytes for GspFmcBootParams {} 986 987 impl GspFmcBootParams { 988 pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { 989 #[allow(non_snake_case)] 990 let init = init!(Self { 991 // Blackwell FSP obtains WPR info from other sources, so 992 // wprCarveoutOffset and wprCarveoutSize are left zero. 993 bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, 994 wpr_meta_addr), 995 gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), 996 ..Zeroable::init_zeroed() 997 }); 998 999 init 1000 } 1001 } 1002