1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 pub(crate) mod commands; 5 mod r570_144; 6 7 // Alias to avoid repeating the version number with every use. 8 use r570_144 as bindings; 9 10 use core::ops::Range; 11 12 use kernel::{ 13 bitfield, 14 dma::{ 15 Coherent, 16 CoherentView, // 17 }, 18 io::{ 19 io_read, 20 io_write, // 21 }, 22 prelude::*, 23 ptr::{ 24 Alignable, 25 Alignment, 26 KnownSize, // 27 }, 28 sizes::{ 29 SizeConstants, 30 SZ_128K, // 31 }, 32 transmute::{ 33 AsBytes, 34 FromBytes, // 35 }, 36 }; 37 38 use crate::{ 39 fb::FbLayout, 40 firmware::gsp::GspFirmware, 41 gpu::{ 42 Architecture, 43 Chipset, // 44 }, 45 gsp::{ 46 cmdq::Cmdq, // 47 GSP_PAGE_SIZE, 48 }, 49 num::{ 50 self, 51 FromSafeCast, // 52 }, 53 }; 54 55 /// Maximum size of a single GSP message queue element in bytes. 56 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 57 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 58 59 /// Empty type to group methods related to heap parameters for running the GSP firmware. 60 enum GspFwHeapParams {} 61 62 /// Minimum required alignment for the GSP heap. 63 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 64 65 impl GspFwHeapParams { 66 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 67 /// and including the first client subdevice allocation). 68 fn base_rm_size(chipset: Chipset) -> u64 { 69 match chipset.arch() { 70 Architecture::Turing | Architecture::Ampere | Architecture::Ada => { 71 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 72 } 73 Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { 74 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 75 } 76 } 77 } 78 79 /// Returns the amount of heap memory required to support a single channel allocation. 80 fn client_alloc_size() -> u64 { 81 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 82 .align_up(GSP_HEAP_ALIGNMENT) 83 .unwrap_or(u64::MAX) 84 } 85 86 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 87 /// `fb_size`. 88 fn management_overhead(fb_size: u64) -> Result<u64> { 89 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 90 91 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 92 .checked_mul(fb_size_gb) 93 .ok_or(EINVAL)? 94 .align_up(GSP_HEAP_ALIGNMENT) 95 .ok_or(EINVAL) 96 } 97 } 98 99 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 100 pub(crate) struct LibosParams { 101 /// The base amount of heap required by the GSP operating system, in bytes. 102 carveout_size: u64, 103 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 104 allowed_heap_size: Range<u64>, 105 } 106 107 impl LibosParams { 108 /// Version 2 of the GSP LIBOS (Turing and GA100) 109 const LIBOS2: LibosParams = LibosParams { 110 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 111 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 112 * u64::SZ_1M 113 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 114 }; 115 116 /// Version 3 of the GSP LIBOS (GA102+) 117 const LIBOS3: LibosParams = LibosParams { 118 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 119 allowed_heap_size: num::u32_as_u64( 120 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 121 ) * u64::SZ_1M 122 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 123 * u64::SZ_1M, 124 }; 125 126 /// Returns the libos parameters corresponding to `chipset`. 127 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 128 if chipset < Chipset::GA102 { 129 &Self::LIBOS2 130 } else { 131 &Self::LIBOS3 132 } 133 } 134 135 /// Returns the WPR heap size to reserve when vGPU is enabled. 136 pub(crate) fn vgpu_wpr_heap_size() -> u64 { 137 u64::from(bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT) 138 } 139 140 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 141 /// of `fb_size` (in bytes) for `chipset`. 142 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 143 // The WPR heap will contain the following: 144 // LIBOS carveout, 145 Ok(self 146 .carveout_size 147 // RM boot working memory, 148 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 149 // One RM client, 150 .saturating_add(GspFwHeapParams::client_alloc_size()) 151 // Overhead for memory management. 152 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 153 // Clamp to the supported heap sizes. 154 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 155 } 156 } 157 158 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 159 /// addresses of the GSP bootloader and firmware. 160 #[repr(transparent)] 161 pub(crate) struct GspFwWprMeta { 162 inner: bindings::GspFwWprMeta, 163 } 164 165 // SAFETY: Padding is explicit and does not contain uninitialized data. 166 unsafe impl AsBytes for GspFwWprMeta {} 167 168 // SAFETY: This struct only contains integer types for which all bit patterns 169 // are valid. 170 unsafe impl FromBytes for GspFwWprMeta {} 171 172 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 173 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 174 175 impl GspFwWprMeta { 176 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 177 /// `fb_layout` layout. 178 pub(crate) fn new<'a>( 179 gsp_firmware: &'a GspFirmware, 180 fb_layout: &'a FbLayout, 181 ) -> impl Init<Self> + 'a { 182 let init_inner = init!(bindings::GspFwWprMeta { 183 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 184 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 185 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 186 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), 187 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 188 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), 189 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 190 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 191 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 192 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 193 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 194 __bindgen_anon_1: GspFwWprMetaBootInfo { 195 sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), 196 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 197 }, 198 }, 199 gspFwRsvdStart: fb_layout.heap.start, 200 nonWprHeapOffset: fb_layout.heap.start, 201 nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, 202 gspFwWprStart: fb_layout.wpr2.start, 203 gspFwHeapOffset: fb_layout.wpr2_heap.start, 204 gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, 205 gspFwOffset: fb_layout.elf.start, 206 bootBinOffset: fb_layout.boot.start, 207 frtsOffset: fb_layout.frts.start, 208 frtsSize: fb_layout.frts.end - fb_layout.frts.start, 209 gspFwWprEnd: fb_layout 210 .vga_workspace 211 .start 212 .align_down(Alignment::new::<SZ_128K>()), 213 gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, 214 fbSize: fb_layout.fb.end - fb_layout.fb.start, 215 vgaWorkspaceOffset: fb_layout.vga_workspace.start, 216 vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, 217 pmuReservedSize: fb_layout.pmu_reserved_size, 218 ..Zeroable::init_zeroed() 219 }); 220 221 init!(GspFwWprMeta { 222 inner <- init_inner, 223 }) 224 } 225 } 226 227 #[derive(Copy, Clone, Debug, PartialEq)] 228 #[repr(u32)] 229 pub(crate) enum MsgFunction { 230 // Common function codes 231 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 232 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 233 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 234 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 235 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 236 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 237 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 238 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 239 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 240 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 241 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 242 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 243 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 244 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 245 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 246 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 247 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 248 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 249 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 250 UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, 251 252 // Event codes 253 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 254 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 255 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 256 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 257 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 258 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 259 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 260 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 261 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 262 } 263 264 impl TryFrom<u32> for MsgFunction { 265 type Error = kernel::error::Error; 266 267 fn try_from(value: u32) -> Result<MsgFunction> { 268 match value { 269 // Common function codes 270 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 271 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 272 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 273 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 274 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 275 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 276 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 277 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 278 Ok(MsgFunction::ContinuationRecord) 279 } 280 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 281 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 282 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 283 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 284 Ok(MsgFunction::GspInitPostObjGpu) 285 } 286 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 287 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 288 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 289 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 290 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 291 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 292 Ok(MsgFunction::SetGuestSystemInfo) 293 } 294 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 295 bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { 296 Ok(MsgFunction::UnloadingGuestDriver) 297 } 298 299 // Event codes 300 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 301 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 302 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 303 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 304 Ok(MsgFunction::GspRunCpuSequencer) 305 } 306 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 307 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 308 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 309 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 310 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 311 _ => Err(EINVAL), 312 } 313 } 314 } 315 316 impl From<MsgFunction> for u32 { 317 fn from(value: MsgFunction) -> Self { 318 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 319 value as u32 320 } 321 } 322 323 /// Sequencer buffer opcode for GSP sequencer commands. 324 #[derive(Copy, Clone, Debug, PartialEq)] 325 #[repr(u32)] 326 pub(crate) enum SeqBufOpcode { 327 // Core operation opcodes 328 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 329 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 330 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 331 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 332 333 // Delay opcode 334 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 335 336 // Register operation opcodes 337 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 338 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 339 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 340 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 341 } 342 343 impl TryFrom<u32> for SeqBufOpcode { 344 type Error = kernel::error::Error; 345 346 fn try_from(value: u32) -> Result<SeqBufOpcode> { 347 match value { 348 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 349 Ok(SeqBufOpcode::CoreReset) 350 } 351 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 352 Ok(SeqBufOpcode::CoreResume) 353 } 354 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 355 Ok(SeqBufOpcode::CoreStart) 356 } 357 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 358 Ok(SeqBufOpcode::CoreWaitForHalt) 359 } 360 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 361 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 362 Ok(SeqBufOpcode::RegModify) 363 } 364 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 365 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 366 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 367 _ => Err(EINVAL), 368 } 369 } 370 } 371 372 impl From<SeqBufOpcode> for u32 { 373 fn from(value: SeqBufOpcode) -> Self { 374 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 375 value as u32 376 } 377 } 378 379 /// Wrapper for GSP sequencer register write payload. 380 #[repr(transparent)] 381 #[derive(Copy, Clone, Debug)] 382 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 383 384 impl RegWritePayload { 385 /// Returns the register address. 386 pub(crate) fn addr(&self) -> u32 { 387 self.0.addr 388 } 389 390 /// Returns the value to write. 391 pub(crate) fn val(&self) -> u32 { 392 self.0.val 393 } 394 } 395 396 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 397 unsafe impl FromBytes for RegWritePayload {} 398 399 // SAFETY: Padding is explicit and will not contain uninitialized data. 400 unsafe impl AsBytes for RegWritePayload {} 401 402 /// Wrapper for GSP sequencer register modify payload. 403 #[repr(transparent)] 404 #[derive(Copy, Clone, Debug)] 405 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 406 407 impl RegModifyPayload { 408 /// Returns the register address. 409 pub(crate) fn addr(&self) -> u32 { 410 self.0.addr 411 } 412 413 /// Returns the mask to apply. 414 pub(crate) fn mask(&self) -> u32 { 415 self.0.mask 416 } 417 418 /// Returns the value to write. 419 pub(crate) fn val(&self) -> u32 { 420 self.0.val 421 } 422 } 423 424 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 425 unsafe impl FromBytes for RegModifyPayload {} 426 427 // SAFETY: Padding is explicit and will not contain uninitialized data. 428 unsafe impl AsBytes for RegModifyPayload {} 429 430 /// Wrapper for GSP sequencer register poll payload. 431 #[repr(transparent)] 432 #[derive(Copy, Clone, Debug)] 433 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 434 435 impl RegPollPayload { 436 /// Returns the register address. 437 pub(crate) fn addr(&self) -> u32 { 438 self.0.addr 439 } 440 441 /// Returns the mask to apply. 442 pub(crate) fn mask(&self) -> u32 { 443 self.0.mask 444 } 445 446 /// Returns the expected value. 447 pub(crate) fn val(&self) -> u32 { 448 self.0.val 449 } 450 451 /// Returns the timeout in microseconds. 452 pub(crate) fn timeout(&self) -> u32 { 453 self.0.timeout 454 } 455 } 456 457 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 458 unsafe impl FromBytes for RegPollPayload {} 459 460 // SAFETY: Padding is explicit and will not contain uninitialized data. 461 unsafe impl AsBytes for RegPollPayload {} 462 463 /// Wrapper for GSP sequencer delay payload. 464 #[repr(transparent)] 465 #[derive(Copy, Clone, Debug)] 466 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 467 468 impl DelayUsPayload { 469 /// Returns the delay value in microseconds. 470 pub(crate) fn val(&self) -> u32 { 471 self.0.val 472 } 473 } 474 475 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 476 unsafe impl FromBytes for DelayUsPayload {} 477 478 // SAFETY: Padding is explicit and will not contain uninitialized data. 479 unsafe impl AsBytes for DelayUsPayload {} 480 481 /// Wrapper for GSP sequencer register store payload. 482 #[repr(transparent)] 483 #[derive(Copy, Clone, Debug)] 484 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 485 486 impl RegStorePayload { 487 /// Returns the register address. 488 pub(crate) fn addr(&self) -> u32 { 489 self.0.addr 490 } 491 492 /// Returns the storage index. 493 #[allow(unused)] 494 pub(crate) fn index(&self) -> u32 { 495 self.0.index 496 } 497 } 498 499 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 500 unsafe impl FromBytes for RegStorePayload {} 501 502 // SAFETY: Padding is explicit and will not contain uninitialized data. 503 unsafe impl AsBytes for RegStorePayload {} 504 505 /// Wrapper for GSP sequencer buffer command. 506 #[repr(transparent)] 507 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 508 509 impl SequencerBufferCmd { 510 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 511 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 512 self.0.opCode.try_into() 513 } 514 515 /// Returns the register write payload by value. 516 /// 517 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 518 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 519 if self.opcode()? != SeqBufOpcode::RegWrite { 520 return Err(EINVAL); 521 } 522 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 523 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 524 } 525 526 /// Returns the register modify payload by value. 527 /// 528 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 529 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 530 if self.opcode()? != SeqBufOpcode::RegModify { 531 return Err(EINVAL); 532 } 533 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 534 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 535 } 536 537 /// Returns the register poll payload by value. 538 /// 539 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 540 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 541 if self.opcode()? != SeqBufOpcode::RegPoll { 542 return Err(EINVAL); 543 } 544 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 545 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 546 } 547 548 /// Returns the delay payload by value. 549 /// 550 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 551 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 552 if self.opcode()? != SeqBufOpcode::DelayUs { 553 return Err(EINVAL); 554 } 555 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 556 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 557 } 558 559 /// Returns the register store payload by value. 560 /// 561 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 562 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 563 if self.opcode()? != SeqBufOpcode::RegStore { 564 return Err(EINVAL); 565 } 566 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 567 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 568 } 569 } 570 571 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 572 unsafe impl FromBytes for SequencerBufferCmd {} 573 574 // SAFETY: Padding is explicit and will not contain uninitialized data. 575 unsafe impl AsBytes for SequencerBufferCmd {} 576 577 /// Wrapper for GSP run CPU sequencer RPC. 578 #[repr(transparent)] 579 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 580 581 impl RunCpuSequencer { 582 /// Returns the command index. 583 pub(crate) fn cmd_index(&self) -> u32 { 584 self.0.cmdIndex 585 } 586 } 587 588 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 589 unsafe impl FromBytes for RunCpuSequencer {} 590 591 // SAFETY: Padding is explicit and will not contain uninitialized data. 592 unsafe impl AsBytes for RunCpuSequencer {} 593 594 /// Struct containing the arguments required to pass a memory buffer to the GSP 595 /// for use during initialisation. 596 /// 597 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 598 /// configured for a larger page size (e.g. 64K pages), we need to give 599 /// the GSP an array of 4K pages. Since we only create physically contiguous 600 /// buffers the math to calculate the addresses is simple. 601 /// 602 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 603 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 604 /// buffers to be physically contiguous anyway. 605 /// 606 /// The memory allocated for the arguments must remain until the GSP sends the 607 /// init_done RPC. 608 #[repr(transparent)] 609 pub(crate) struct LibosMemoryRegionInitArgument { 610 inner: bindings::LibosMemoryRegionInitArgument, 611 } 612 613 // SAFETY: Padding is explicit and does not contain uninitialized data. 614 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 615 616 // SAFETY: This struct only contains integer types for which all bit patterns 617 // are valid. 618 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 619 620 impl LibosMemoryRegionInitArgument { 621 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 622 name: &'static str, 623 obj: &'a Coherent<A>, 624 ) -> impl Init<Self> + 'a { 625 /// Generates the `ID8` identifier required for some GSP objects. 626 fn id8(name: &str) -> u64 { 627 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 628 629 for (c, b) in name.bytes().rev().zip(&mut bytes) { 630 *b = c; 631 } 632 633 u64::from_ne_bytes(bytes) 634 } 635 636 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 637 id8: id8(name), 638 pa: obj.dma_handle(), 639 size: num::usize_as_u64(obj.size()), 640 kind: num::u32_into_u8::< 641 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 642 >(), 643 loc: num::u32_into_u8::< 644 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 645 >(), 646 ..Zeroable::init_zeroed() 647 }); 648 649 init!(LibosMemoryRegionInitArgument { 650 inner <- init_inner, 651 }) 652 } 653 } 654 655 /// TX header for setting up a message queue with the GSP. 656 #[repr(transparent)] 657 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 658 659 impl MsgqTxHeader { 660 /// Create a new TX queue header. 661 /// 662 /// # Arguments 663 /// 664 /// * `msgq_size` - Total size of the message queue structure, in bytes. 665 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 666 /// structure. 667 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 668 /// allocated for the message queue in the message queue structure. 669 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 670 Self(bindings::msgqTxHeader { 671 version: 0, 672 size: msgq_size, 673 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 674 msgCount: msg_count, 675 writePtr: 0, 676 flags: 1, 677 rxHdrOff: rx_hdr_offset, 678 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 679 }) 680 } 681 682 /// Returns the value of the write pointer for this queue. 683 pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { 684 io_read!(this, .0.writePtr) 685 } 686 687 /// Sets the value of the write pointer for this queue. 688 pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { 689 io_write!(this, .0.writePtr, val) 690 } 691 } 692 693 // SAFETY: Padding is explicit and does not contain uninitialized data. 694 unsafe impl AsBytes for MsgqTxHeader {} 695 696 /// RX header for setting up a message queue with the GSP. 697 #[repr(transparent)] 698 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 699 700 /// Header for the message RX queue. 701 impl MsgqRxHeader { 702 /// Creates a new RX queue header. 703 pub(crate) fn new() -> Self { 704 Self(Default::default()) 705 } 706 707 /// Returns the value of the read pointer for this queue. 708 pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { 709 io_read!(this, .0.readPtr) 710 } 711 712 /// Sets the value of the read pointer for this queue. 713 pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { 714 io_write!(this, .0.readPtr, val) 715 } 716 } 717 718 // SAFETY: Padding is explicit and does not contain uninitialized data. 719 unsafe impl AsBytes for MsgqRxHeader {} 720 721 bitfield! { 722 struct MsgHeaderVersion(u32) { 723 31:24 major; 724 23:16 minor; 725 } 726 } 727 728 impl MsgHeaderVersion { 729 const MAJOR_TOT: u8 = 3; 730 const MINOR_TOT: u8 = 0; 731 732 fn new() -> Self { 733 Self::zeroed() 734 .with_major(Self::MAJOR_TOT) 735 .with_minor(Self::MINOR_TOT) 736 } 737 } 738 739 impl bindings::rpc_message_header_v { 740 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 741 type RpcMessageHeader = bindings::rpc_message_header_v; 742 743 try_init!(RpcMessageHeader { 744 header_version: MsgHeaderVersion::new().into(), 745 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 746 function: function.into(), 747 length: size_of::<Self>() 748 .checked_add(cmd_size) 749 .ok_or(EOVERFLOW) 750 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 751 rpc_result: 0xffffffff, 752 rpc_result_private: 0xffffffff, 753 ..Zeroable::init_zeroed() 754 }) 755 } 756 } 757 758 /// GSP Message Element. 759 /// 760 /// This is essentially a message header expected to be followed by the message data. 761 #[repr(transparent)] 762 pub(crate) struct GspMsgElement { 763 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 764 } 765 766 impl GspMsgElement { 767 /// Creates a new message element. 768 /// 769 /// # Arguments 770 /// 771 /// * `sequence` - Sequence number of the message. 772 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 773 /// * `function` - Function of the message. 774 pub(crate) fn init( 775 sequence: u32, 776 cmd_size: usize, 777 function: MsgFunction, 778 ) -> impl Init<Self, Error> { 779 type RpcMessageHeader = bindings::rpc_message_header_v; 780 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 781 let init_inner = try_init!(InnerGspMsgElement { 782 seqNum: sequence, 783 elemCount: size_of::<Self>() 784 .checked_add(cmd_size) 785 .ok_or(EOVERFLOW)? 786 .div_ceil(GSP_PAGE_SIZE) 787 .try_into() 788 .map_err(|_| EOVERFLOW)?, 789 rpc <- RpcMessageHeader::init(cmd_size, function), 790 ..Zeroable::init_zeroed() 791 }); 792 793 try_init!(GspMsgElement { 794 inner <- init_inner, 795 }) 796 } 797 798 /// Sets the checksum of this message. 799 /// 800 /// Since the header is also part of the checksum, this is usually called after the whole 801 /// message has been written to the shared memory area. 802 pub(crate) fn set_checksum(&mut self, checksum: u32) { 803 self.inner.checkSum = checksum; 804 } 805 806 /// Returns the length of the message's payload. 807 pub(crate) fn payload_length(&self) -> usize { 808 // `rpc.length` includes the length of the RPC message header. 809 num::u32_as_usize(self.inner.rpc.length) 810 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 811 } 812 813 /// Returns the total length of the message, message and RPC headers included. 814 pub(crate) fn length(&self) -> usize { 815 size_of::<Self>() + self.payload_length() 816 } 817 818 // Returns the sequence number of the message. 819 pub(crate) fn sequence(&self) -> u32 { 820 self.inner.rpc.sequence 821 } 822 823 // Returns the function of the message, if it is valid, or the invalid function number as an 824 // error. 825 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 826 self.inner 827 .rpc 828 .function 829 .try_into() 830 .map_err(|_| self.inner.rpc.function) 831 } 832 833 // Returns the number of elements (i.e. memory pages) used by this message. 834 pub(crate) fn element_count(&self) -> u32 { 835 self.inner.elemCount 836 } 837 } 838 839 // SAFETY: Padding is explicit and does not contain uninitialized data. 840 unsafe impl AsBytes for GspMsgElement {} 841 842 // SAFETY: This struct only contains integer types for which all bit patterns 843 // are valid. 844 unsafe impl FromBytes for GspMsgElement {} 845 846 /// Arguments for GSP startup. 847 #[repr(transparent)] 848 #[derive(Zeroable)] 849 pub(crate) struct GspArgumentsCached { 850 inner: bindings::GSP_ARGUMENTS_CACHED, 851 } 852 853 impl GspArgumentsCached { 854 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 855 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 856 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 857 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 858 bDmemStack: 1, 859 ..Zeroable::init_zeroed() 860 }); 861 862 init!(GspArgumentsCached { 863 inner <- init_inner, 864 }) 865 } 866 } 867 868 // SAFETY: Padding is explicit and will not contain uninitialized data. 869 unsafe impl AsBytes for GspArgumentsCached {} 870 871 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 872 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 873 /// to that size. 874 #[repr(C)] 875 #[derive(Zeroable)] 876 pub(crate) struct GspArgumentsPadded { 877 pub(crate) inner: GspArgumentsCached, 878 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 879 } 880 881 impl GspArgumentsPadded { 882 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 883 init!(GspArgumentsPadded { 884 inner <- GspArgumentsCached::new(cmdq), 885 ..Zeroable::init_zeroed() 886 }) 887 } 888 } 889 890 // SAFETY: Padding is explicit and will not contain uninitialized data. 891 unsafe impl AsBytes for GspArgumentsPadded {} 892 893 // SAFETY: This struct only contains integer types for which all bit patterns 894 // are valid. 895 unsafe impl FromBytes for GspArgumentsPadded {} 896 897 /// Init arguments for the message queue. 898 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 899 900 impl MessageQueueInitArguments { 901 /// Creates a new init arguments structure for `cmdq`. 902 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 903 init!(MessageQueueInitArguments { 904 sharedMemPhysAddr: cmdq.dma_handle, 905 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 906 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 907 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 908 ..Zeroable::init_zeroed() 909 }) 910 } 911 } 912 913 #[repr(u32)] 914 pub(crate) enum GspDmaTarget { 915 #[expect(dead_code)] 916 LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, 917 CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, 918 NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, 919 } 920 921 type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; 922 923 impl GspAcrBootGspRmParams { 924 fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { 925 let params = init!(Self { 926 target: target as u32, 927 gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), 928 gspRmDescOffset: wpr_meta_addr, 929 bIsGspRmBoot: 1, 930 wprCarveoutOffset: 0, 931 wprCarveoutSize: 0, 932 __bindgen_padding_0: Default::default(), 933 }); 934 935 params 936 } 937 } 938 939 type GspRmParams = bindings::GSP_RM_PARAMS; 940 941 impl GspRmParams { 942 fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { 943 let params = init!(Self { 944 target: target as u32, 945 bootArgsOffset: libos_addr, 946 __bindgen_padding_0: Default::default(), 947 }); 948 949 params 950 } 951 } 952 953 pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; 954 955 // SAFETY: Padding is explicit and will not contain uninitialized data. 956 unsafe impl AsBytes for GspFmcBootParams {} 957 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 958 unsafe impl FromBytes for GspFmcBootParams {} 959 960 impl GspFmcBootParams { 961 pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { 962 let init = init!(Self { 963 // Blackwell FSP obtains WPR info from other sources, so 964 // wprCarveoutOffset and wprCarveoutSize are left zero. 965 bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, 966 wpr_meta_addr), 967 gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), 968 ..Zeroable::init_zeroed() 969 }); 970 971 init 972 } 973 } 974