1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 pub(crate) mod commands; 5 mod r570_144; 6 7 // Alias to avoid repeating the version number with every use. 8 use r570_144 as bindings; 9 10 use core::ops::Range; 11 12 use kernel::{ 13 dma::{ 14 Coherent, 15 CoherentView, // 16 }, 17 io::{ 18 io_read, 19 io_write, // 20 }, 21 prelude::*, 22 ptr::{ 23 Alignable, 24 Alignment, 25 KnownSize, // 26 }, 27 sizes::{ 28 SizeConstants, 29 SZ_128K, // 30 }, 31 transmute::{ 32 AsBytes, 33 FromBytes, // 34 }, 35 }; 36 37 use crate::{ 38 fb::FbLayout, 39 firmware::gsp::GspFirmware, 40 gpu::{ 41 Architecture, 42 Chipset, // 43 }, 44 gsp::{ 45 cmdq::Cmdq, // 46 GSP_PAGE_SIZE, 47 }, 48 num::{ 49 self, 50 FromSafeCast, // 51 }, 52 }; 53 54 /// Maximum size of a single GSP message queue element in bytes. 55 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 56 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 57 58 /// Empty type to group methods related to heap parameters for running the GSP firmware. 59 enum GspFwHeapParams {} 60 61 /// Minimum required alignment for the GSP heap. 62 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 63 64 impl GspFwHeapParams { 65 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 66 /// and including the first client subdevice allocation). 67 fn base_rm_size(chipset: Chipset) -> u64 { 68 match chipset.arch() { 69 Architecture::Turing | Architecture::Ampere | Architecture::Ada => { 70 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 71 } 72 Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { 73 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 74 } 75 } 76 } 77 78 /// Returns the amount of heap memory required to support a single channel allocation. 79 fn client_alloc_size() -> u64 { 80 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 81 .align_up(GSP_HEAP_ALIGNMENT) 82 .unwrap_or(u64::MAX) 83 } 84 85 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 86 /// `fb_size`. 87 fn management_overhead(fb_size: u64) -> Result<u64> { 88 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 89 90 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 91 .checked_mul(fb_size_gb) 92 .ok_or(EINVAL)? 93 .align_up(GSP_HEAP_ALIGNMENT) 94 .ok_or(EINVAL) 95 } 96 } 97 98 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 99 pub(crate) struct LibosParams { 100 /// The base amount of heap required by the GSP operating system, in bytes. 101 carveout_size: u64, 102 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 103 allowed_heap_size: Range<u64>, 104 } 105 106 impl LibosParams { 107 /// Version 2 of the GSP LIBOS (Turing and GA100) 108 const LIBOS2: LibosParams = LibosParams { 109 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 110 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 111 * u64::SZ_1M 112 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 113 }; 114 115 /// Version 3 of the GSP LIBOS (GA102+) 116 const LIBOS3: LibosParams = LibosParams { 117 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 118 allowed_heap_size: num::u32_as_u64( 119 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 120 ) * u64::SZ_1M 121 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 122 * u64::SZ_1M, 123 }; 124 125 /// Returns the libos parameters corresponding to `chipset`. 126 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 127 if chipset < Chipset::GA102 { 128 &Self::LIBOS2 129 } else { 130 &Self::LIBOS3 131 } 132 } 133 134 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 135 /// of `fb_size` (in bytes) for `chipset`. 136 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 137 // The WPR heap will contain the following: 138 // LIBOS carveout, 139 Ok(self 140 .carveout_size 141 // RM boot working memory, 142 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 143 // One RM client, 144 .saturating_add(GspFwHeapParams::client_alloc_size()) 145 // Overhead for memory management. 146 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 147 // Clamp to the supported heap sizes. 148 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 149 } 150 } 151 152 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 153 /// addresses of the GSP bootloader and firmware. 154 #[repr(transparent)] 155 pub(crate) struct GspFwWprMeta { 156 inner: bindings::GspFwWprMeta, 157 } 158 159 // SAFETY: Padding is explicit and does not contain uninitialized data. 160 unsafe impl AsBytes for GspFwWprMeta {} 161 162 // SAFETY: This struct only contains integer types for which all bit patterns 163 // are valid. 164 unsafe impl FromBytes for GspFwWprMeta {} 165 166 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 167 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 168 169 impl GspFwWprMeta { 170 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 171 /// `fb_layout` layout. 172 pub(crate) fn new<'a>( 173 gsp_firmware: &'a GspFirmware, 174 fb_layout: &'a FbLayout, 175 ) -> impl Init<Self> + 'a { 176 #[allow(non_snake_case)] 177 let init_inner = init!(bindings::GspFwWprMeta { 178 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 179 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 180 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 181 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), 182 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 183 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), 184 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 185 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 186 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 187 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 188 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 189 __bindgen_anon_1: GspFwWprMetaBootInfo { 190 sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), 191 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 192 }, 193 }, 194 gspFwRsvdStart: fb_layout.heap.start, 195 nonWprHeapOffset: fb_layout.heap.start, 196 nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, 197 gspFwWprStart: fb_layout.wpr2.start, 198 gspFwHeapOffset: fb_layout.wpr2_heap.start, 199 gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, 200 gspFwOffset: fb_layout.elf.start, 201 bootBinOffset: fb_layout.boot.start, 202 frtsOffset: fb_layout.frts.start, 203 frtsSize: fb_layout.frts.end - fb_layout.frts.start, 204 gspFwWprEnd: fb_layout 205 .vga_workspace 206 .start 207 .align_down(Alignment::new::<SZ_128K>()), 208 gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, 209 fbSize: fb_layout.fb.end - fb_layout.fb.start, 210 vgaWorkspaceOffset: fb_layout.vga_workspace.start, 211 vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, 212 pmuReservedSize: fb_layout.pmu_reserved_size, 213 ..Zeroable::init_zeroed() 214 }); 215 216 init!(GspFwWprMeta { 217 inner <- init_inner, 218 }) 219 } 220 } 221 222 #[derive(Copy, Clone, Debug, PartialEq)] 223 #[repr(u32)] 224 pub(crate) enum MsgFunction { 225 // Common function codes 226 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 227 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 228 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 229 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 230 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 231 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 232 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 233 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 234 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 235 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 236 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 237 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 238 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 239 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 240 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 241 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 242 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 243 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 244 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 245 UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, 246 247 // Event codes 248 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 249 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 250 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 251 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 252 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 253 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 254 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 255 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 256 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 257 } 258 259 impl TryFrom<u32> for MsgFunction { 260 type Error = kernel::error::Error; 261 262 fn try_from(value: u32) -> Result<MsgFunction> { 263 match value { 264 // Common function codes 265 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 266 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 267 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 268 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 269 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 270 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 271 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 272 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 273 Ok(MsgFunction::ContinuationRecord) 274 } 275 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 276 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 277 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 278 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 279 Ok(MsgFunction::GspInitPostObjGpu) 280 } 281 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 282 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 283 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 284 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 285 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 286 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 287 Ok(MsgFunction::SetGuestSystemInfo) 288 } 289 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 290 bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { 291 Ok(MsgFunction::UnloadingGuestDriver) 292 } 293 294 // Event codes 295 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 296 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 297 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 298 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 299 Ok(MsgFunction::GspRunCpuSequencer) 300 } 301 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 302 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 303 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 304 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 305 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 306 _ => Err(EINVAL), 307 } 308 } 309 } 310 311 impl From<MsgFunction> for u32 { 312 fn from(value: MsgFunction) -> Self { 313 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 314 value as u32 315 } 316 } 317 318 /// Sequencer buffer opcode for GSP sequencer commands. 319 #[derive(Copy, Clone, Debug, PartialEq)] 320 #[repr(u32)] 321 pub(crate) enum SeqBufOpcode { 322 // Core operation opcodes 323 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 324 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 325 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 326 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 327 328 // Delay opcode 329 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 330 331 // Register operation opcodes 332 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 333 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 334 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 335 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 336 } 337 338 impl TryFrom<u32> for SeqBufOpcode { 339 type Error = kernel::error::Error; 340 341 fn try_from(value: u32) -> Result<SeqBufOpcode> { 342 match value { 343 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 344 Ok(SeqBufOpcode::CoreReset) 345 } 346 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 347 Ok(SeqBufOpcode::CoreResume) 348 } 349 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 350 Ok(SeqBufOpcode::CoreStart) 351 } 352 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 353 Ok(SeqBufOpcode::CoreWaitForHalt) 354 } 355 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 356 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 357 Ok(SeqBufOpcode::RegModify) 358 } 359 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 360 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 361 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 362 _ => Err(EINVAL), 363 } 364 } 365 } 366 367 impl From<SeqBufOpcode> for u32 { 368 fn from(value: SeqBufOpcode) -> Self { 369 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 370 value as u32 371 } 372 } 373 374 /// Wrapper for GSP sequencer register write payload. 375 #[repr(transparent)] 376 #[derive(Copy, Clone, Debug)] 377 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 378 379 impl RegWritePayload { 380 /// Returns the register address. 381 pub(crate) fn addr(&self) -> u32 { 382 self.0.addr 383 } 384 385 /// Returns the value to write. 386 pub(crate) fn val(&self) -> u32 { 387 self.0.val 388 } 389 } 390 391 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 392 unsafe impl FromBytes for RegWritePayload {} 393 394 // SAFETY: Padding is explicit and will not contain uninitialized data. 395 unsafe impl AsBytes for RegWritePayload {} 396 397 /// Wrapper for GSP sequencer register modify payload. 398 #[repr(transparent)] 399 #[derive(Copy, Clone, Debug)] 400 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 401 402 impl RegModifyPayload { 403 /// Returns the register address. 404 pub(crate) fn addr(&self) -> u32 { 405 self.0.addr 406 } 407 408 /// Returns the mask to apply. 409 pub(crate) fn mask(&self) -> u32 { 410 self.0.mask 411 } 412 413 /// Returns the value to write. 414 pub(crate) fn val(&self) -> u32 { 415 self.0.val 416 } 417 } 418 419 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 420 unsafe impl FromBytes for RegModifyPayload {} 421 422 // SAFETY: Padding is explicit and will not contain uninitialized data. 423 unsafe impl AsBytes for RegModifyPayload {} 424 425 /// Wrapper for GSP sequencer register poll payload. 426 #[repr(transparent)] 427 #[derive(Copy, Clone, Debug)] 428 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 429 430 impl RegPollPayload { 431 /// Returns the register address. 432 pub(crate) fn addr(&self) -> u32 { 433 self.0.addr 434 } 435 436 /// Returns the mask to apply. 437 pub(crate) fn mask(&self) -> u32 { 438 self.0.mask 439 } 440 441 /// Returns the expected value. 442 pub(crate) fn val(&self) -> u32 { 443 self.0.val 444 } 445 446 /// Returns the timeout in microseconds. 447 pub(crate) fn timeout(&self) -> u32 { 448 self.0.timeout 449 } 450 } 451 452 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 453 unsafe impl FromBytes for RegPollPayload {} 454 455 // SAFETY: Padding is explicit and will not contain uninitialized data. 456 unsafe impl AsBytes for RegPollPayload {} 457 458 /// Wrapper for GSP sequencer delay payload. 459 #[repr(transparent)] 460 #[derive(Copy, Clone, Debug)] 461 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 462 463 impl DelayUsPayload { 464 /// Returns the delay value in microseconds. 465 pub(crate) fn val(&self) -> u32 { 466 self.0.val 467 } 468 } 469 470 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 471 unsafe impl FromBytes for DelayUsPayload {} 472 473 // SAFETY: Padding is explicit and will not contain uninitialized data. 474 unsafe impl AsBytes for DelayUsPayload {} 475 476 /// Wrapper for GSP sequencer register store payload. 477 #[repr(transparent)] 478 #[derive(Copy, Clone, Debug)] 479 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 480 481 impl RegStorePayload { 482 /// Returns the register address. 483 pub(crate) fn addr(&self) -> u32 { 484 self.0.addr 485 } 486 487 /// Returns the storage index. 488 #[allow(unused)] 489 pub(crate) fn index(&self) -> u32 { 490 self.0.index 491 } 492 } 493 494 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 495 unsafe impl FromBytes for RegStorePayload {} 496 497 // SAFETY: Padding is explicit and will not contain uninitialized data. 498 unsafe impl AsBytes for RegStorePayload {} 499 500 /// Wrapper for GSP sequencer buffer command. 501 #[repr(transparent)] 502 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 503 504 impl SequencerBufferCmd { 505 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 506 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 507 self.0.opCode.try_into() 508 } 509 510 /// Returns the register write payload by value. 511 /// 512 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 513 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 514 if self.opcode()? != SeqBufOpcode::RegWrite { 515 return Err(EINVAL); 516 } 517 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 518 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 519 } 520 521 /// Returns the register modify payload by value. 522 /// 523 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 524 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 525 if self.opcode()? != SeqBufOpcode::RegModify { 526 return Err(EINVAL); 527 } 528 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 529 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 530 } 531 532 /// Returns the register poll payload by value. 533 /// 534 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 535 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 536 if self.opcode()? != SeqBufOpcode::RegPoll { 537 return Err(EINVAL); 538 } 539 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 540 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 541 } 542 543 /// Returns the delay payload by value. 544 /// 545 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 546 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 547 if self.opcode()? != SeqBufOpcode::DelayUs { 548 return Err(EINVAL); 549 } 550 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 551 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 552 } 553 554 /// Returns the register store payload by value. 555 /// 556 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 557 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 558 if self.opcode()? != SeqBufOpcode::RegStore { 559 return Err(EINVAL); 560 } 561 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 562 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 563 } 564 } 565 566 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 567 unsafe impl FromBytes for SequencerBufferCmd {} 568 569 // SAFETY: Padding is explicit and will not contain uninitialized data. 570 unsafe impl AsBytes for SequencerBufferCmd {} 571 572 /// Wrapper for GSP run CPU sequencer RPC. 573 #[repr(transparent)] 574 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 575 576 impl RunCpuSequencer { 577 /// Returns the command index. 578 pub(crate) fn cmd_index(&self) -> u32 { 579 self.0.cmdIndex 580 } 581 } 582 583 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 584 unsafe impl FromBytes for RunCpuSequencer {} 585 586 // SAFETY: Padding is explicit and will not contain uninitialized data. 587 unsafe impl AsBytes for RunCpuSequencer {} 588 589 /// Struct containing the arguments required to pass a memory buffer to the GSP 590 /// for use during initialisation. 591 /// 592 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 593 /// configured for a larger page size (e.g. 64K pages), we need to give 594 /// the GSP an array of 4K pages. Since we only create physically contiguous 595 /// buffers the math to calculate the addresses is simple. 596 /// 597 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 598 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 599 /// buffers to be physically contiguous anyway. 600 /// 601 /// The memory allocated for the arguments must remain until the GSP sends the 602 /// init_done RPC. 603 #[repr(transparent)] 604 pub(crate) struct LibosMemoryRegionInitArgument { 605 inner: bindings::LibosMemoryRegionInitArgument, 606 } 607 608 // SAFETY: Padding is explicit and does not contain uninitialized data. 609 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 610 611 // SAFETY: This struct only contains integer types for which all bit patterns 612 // are valid. 613 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 614 615 impl LibosMemoryRegionInitArgument { 616 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 617 name: &'static str, 618 obj: &'a Coherent<A>, 619 ) -> impl Init<Self> + 'a { 620 /// Generates the `ID8` identifier required for some GSP objects. 621 fn id8(name: &str) -> u64 { 622 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 623 624 for (c, b) in name.bytes().rev().zip(&mut bytes) { 625 *b = c; 626 } 627 628 u64::from_ne_bytes(bytes) 629 } 630 631 #[allow(non_snake_case)] 632 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 633 id8: id8(name), 634 pa: obj.dma_handle(), 635 size: num::usize_as_u64(obj.size()), 636 kind: num::u32_into_u8::< 637 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 638 >(), 639 loc: num::u32_into_u8::< 640 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 641 >(), 642 ..Zeroable::init_zeroed() 643 }); 644 645 init!(LibosMemoryRegionInitArgument { 646 inner <- init_inner, 647 }) 648 } 649 } 650 651 /// TX header for setting up a message queue with the GSP. 652 #[repr(transparent)] 653 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 654 655 impl MsgqTxHeader { 656 /// Create a new TX queue header. 657 /// 658 /// # Arguments 659 /// 660 /// * `msgq_size` - Total size of the message queue structure, in bytes. 661 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 662 /// structure. 663 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 664 /// allocated for the message queue in the message queue structure. 665 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 666 Self(bindings::msgqTxHeader { 667 version: 0, 668 size: msgq_size, 669 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 670 msgCount: msg_count, 671 writePtr: 0, 672 flags: 1, 673 rxHdrOff: rx_hdr_offset, 674 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 675 }) 676 } 677 678 /// Returns the value of the write pointer for this queue. 679 pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { 680 io_read!(this, .0.writePtr) 681 } 682 683 /// Sets the value of the write pointer for this queue. 684 pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { 685 io_write!(this, .0.writePtr, val) 686 } 687 } 688 689 // SAFETY: Padding is explicit and does not contain uninitialized data. 690 unsafe impl AsBytes for MsgqTxHeader {} 691 692 /// RX header for setting up a message queue with the GSP. 693 #[repr(transparent)] 694 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 695 696 /// Header for the message RX queue. 697 impl MsgqRxHeader { 698 /// Creates a new RX queue header. 699 pub(crate) fn new() -> Self { 700 Self(Default::default()) 701 } 702 703 /// Returns the value of the read pointer for this queue. 704 pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { 705 io_read!(this, .0.readPtr) 706 } 707 708 /// Sets the value of the read pointer for this queue. 709 pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { 710 io_write!(this, .0.readPtr, val) 711 } 712 } 713 714 // SAFETY: Padding is explicit and does not contain uninitialized data. 715 unsafe impl AsBytes for MsgqRxHeader {} 716 717 bitfield! { 718 struct MsgHeaderVersion(u32) { 719 31:24 major as u8; 720 23:16 minor as u8; 721 } 722 } 723 724 impl MsgHeaderVersion { 725 const MAJOR_TOT: u8 = 3; 726 const MINOR_TOT: u8 = 0; 727 728 fn new() -> Self { 729 Self::default() 730 .set_major(Self::MAJOR_TOT) 731 .set_minor(Self::MINOR_TOT) 732 } 733 } 734 735 impl bindings::rpc_message_header_v { 736 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 737 type RpcMessageHeader = bindings::rpc_message_header_v; 738 739 try_init!(RpcMessageHeader { 740 header_version: MsgHeaderVersion::new().into(), 741 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 742 function: function.into(), 743 length: size_of::<Self>() 744 .checked_add(cmd_size) 745 .ok_or(EOVERFLOW) 746 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 747 rpc_result: 0xffffffff, 748 rpc_result_private: 0xffffffff, 749 ..Zeroable::init_zeroed() 750 }) 751 } 752 } 753 754 /// GSP Message Element. 755 /// 756 /// This is essentially a message header expected to be followed by the message data. 757 #[repr(transparent)] 758 pub(crate) struct GspMsgElement { 759 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 760 } 761 762 impl GspMsgElement { 763 /// Creates a new message element. 764 /// 765 /// # Arguments 766 /// 767 /// * `sequence` - Sequence number of the message. 768 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 769 /// * `function` - Function of the message. 770 #[allow(non_snake_case)] 771 pub(crate) fn init( 772 sequence: u32, 773 cmd_size: usize, 774 function: MsgFunction, 775 ) -> impl Init<Self, Error> { 776 type RpcMessageHeader = bindings::rpc_message_header_v; 777 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 778 let init_inner = try_init!(InnerGspMsgElement { 779 seqNum: sequence, 780 elemCount: size_of::<Self>() 781 .checked_add(cmd_size) 782 .ok_or(EOVERFLOW)? 783 .div_ceil(GSP_PAGE_SIZE) 784 .try_into() 785 .map_err(|_| EOVERFLOW)?, 786 rpc <- RpcMessageHeader::init(cmd_size, function), 787 ..Zeroable::init_zeroed() 788 }); 789 790 try_init!(GspMsgElement { 791 inner <- init_inner, 792 }) 793 } 794 795 /// Sets the checksum of this message. 796 /// 797 /// Since the header is also part of the checksum, this is usually called after the whole 798 /// message has been written to the shared memory area. 799 pub(crate) fn set_checksum(&mut self, checksum: u32) { 800 self.inner.checkSum = checksum; 801 } 802 803 /// Returns the length of the message's payload. 804 pub(crate) fn payload_length(&self) -> usize { 805 // `rpc.length` includes the length of the RPC message header. 806 num::u32_as_usize(self.inner.rpc.length) 807 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 808 } 809 810 /// Returns the total length of the message, message and RPC headers included. 811 pub(crate) fn length(&self) -> usize { 812 size_of::<Self>() + self.payload_length() 813 } 814 815 // Returns the sequence number of the message. 816 pub(crate) fn sequence(&self) -> u32 { 817 self.inner.rpc.sequence 818 } 819 820 // Returns the function of the message, if it is valid, or the invalid function number as an 821 // error. 822 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 823 self.inner 824 .rpc 825 .function 826 .try_into() 827 .map_err(|_| self.inner.rpc.function) 828 } 829 830 // Returns the number of elements (i.e. memory pages) used by this message. 831 pub(crate) fn element_count(&self) -> u32 { 832 self.inner.elemCount 833 } 834 } 835 836 // SAFETY: Padding is explicit and does not contain uninitialized data. 837 unsafe impl AsBytes for GspMsgElement {} 838 839 // SAFETY: This struct only contains integer types for which all bit patterns 840 // are valid. 841 unsafe impl FromBytes for GspMsgElement {} 842 843 /// Arguments for GSP startup. 844 #[repr(transparent)] 845 #[derive(Zeroable)] 846 pub(crate) struct GspArgumentsCached { 847 inner: bindings::GSP_ARGUMENTS_CACHED, 848 } 849 850 impl GspArgumentsCached { 851 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 852 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 853 #[allow(non_snake_case)] 854 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 855 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 856 bDmemStack: 1, 857 ..Zeroable::init_zeroed() 858 }); 859 860 init!(GspArgumentsCached { 861 inner <- init_inner, 862 }) 863 } 864 } 865 866 // SAFETY: Padding is explicit and will not contain uninitialized data. 867 unsafe impl AsBytes for GspArgumentsCached {} 868 869 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 870 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 871 /// to that size. 872 #[repr(C)] 873 #[derive(Zeroable)] 874 pub(crate) struct GspArgumentsPadded { 875 pub(crate) inner: GspArgumentsCached, 876 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 877 } 878 879 impl GspArgumentsPadded { 880 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 881 init!(GspArgumentsPadded { 882 inner <- GspArgumentsCached::new(cmdq), 883 ..Zeroable::init_zeroed() 884 }) 885 } 886 } 887 888 // SAFETY: Padding is explicit and will not contain uninitialized data. 889 unsafe impl AsBytes for GspArgumentsPadded {} 890 891 // SAFETY: This struct only contains integer types for which all bit patterns 892 // are valid. 893 unsafe impl FromBytes for GspArgumentsPadded {} 894 895 /// Init arguments for the message queue. 896 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 897 898 impl MessageQueueInitArguments { 899 /// Creates a new init arguments structure for `cmdq`. 900 #[allow(non_snake_case)] 901 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 902 init!(MessageQueueInitArguments { 903 sharedMemPhysAddr: cmdq.dma_handle, 904 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 905 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 906 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 907 ..Zeroable::init_zeroed() 908 }) 909 } 910 } 911 912 #[repr(u32)] 913 pub(crate) enum GspDmaTarget { 914 #[expect(dead_code)] 915 LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, 916 CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, 917 NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, 918 } 919 920 type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; 921 922 impl GspAcrBootGspRmParams { 923 fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { 924 #[allow(non_snake_case)] 925 let params = init!(Self { 926 target: target as u32, 927 gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), 928 gspRmDescOffset: wpr_meta_addr, 929 bIsGspRmBoot: 1, 930 wprCarveoutOffset: 0, 931 wprCarveoutSize: 0, 932 __bindgen_padding_0: Default::default(), 933 }); 934 935 params 936 } 937 } 938 939 type GspRmParams = bindings::GSP_RM_PARAMS; 940 941 impl GspRmParams { 942 fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { 943 #[allow(non_snake_case)] 944 let params = init!(Self { 945 target: target as u32, 946 bootArgsOffset: libos_addr, 947 __bindgen_padding_0: Default::default(), 948 }); 949 950 params 951 } 952 } 953 954 pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; 955 956 // SAFETY: Padding is explicit and will not contain uninitialized data. 957 unsafe impl AsBytes for GspFmcBootParams {} 958 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 959 unsafe impl FromBytes for GspFmcBootParams {} 960 961 impl GspFmcBootParams { 962 pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { 963 #[allow(non_snake_case)] 964 let init = init!(Self { 965 // Blackwell FSP obtains WPR info from other sources, so 966 // wprCarveoutOffset and wprCarveoutSize are left zero. 967 bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, 968 wpr_meta_addr), 969 gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), 970 ..Zeroable::init_zeroed() 971 }); 972 973 init 974 } 975 } 976