1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 pub(crate) mod commands; 5 mod r570_144; 6 7 // Alias to avoid repeating the version number with every use. 8 use r570_144 as bindings; 9 10 use core::ops::Range; 11 12 use kernel::{ 13 bitfield, 14 dma::{ 15 Coherent, 16 CoherentView, // 17 }, 18 io::{ 19 io_read, 20 io_write, // 21 }, 22 prelude::*, 23 ptr::{ 24 Alignable, 25 Alignment, 26 KnownSize, // 27 }, 28 sizes::{ 29 SizeConstants, 30 SZ_128K, // 31 }, 32 transmute::{ 33 AsBytes, 34 FromBytes, // 35 }, 36 }; 37 38 use crate::{ 39 fb::FbLayout, 40 firmware::gsp::GspFirmware, 41 gpu::{ 42 Architecture, 43 Chipset, // 44 }, 45 gsp::{ 46 cmdq::Cmdq, // 47 GSP_PAGE_SIZE, 48 }, 49 num::{ 50 self, 51 FromSafeCast, // 52 }, 53 }; 54 55 /// Maximum size of a single GSP message queue element in bytes. 56 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 57 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 58 59 /// Empty type to group methods related to heap parameters for running the GSP firmware. 60 enum GspFwHeapParams {} 61 62 /// Minimum required alignment for the GSP heap. 63 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 64 65 impl GspFwHeapParams { 66 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 67 /// and including the first client subdevice allocation). 68 fn base_rm_size(chipset: Chipset) -> u64 { 69 match chipset.arch() { 70 Architecture::Turing | Architecture::Ampere | Architecture::Ada => { 71 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 72 } 73 Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { 74 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 75 } 76 } 77 } 78 79 /// Returns the amount of heap memory required to support a single channel allocation. 80 fn client_alloc_size() -> u64 { 81 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 82 .align_up(GSP_HEAP_ALIGNMENT) 83 .unwrap_or(u64::MAX) 84 } 85 86 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 87 /// `fb_size`. 88 fn management_overhead(fb_size: u64) -> Result<u64> { 89 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 90 91 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 92 .checked_mul(fb_size_gb) 93 .ok_or(EINVAL)? 94 .align_up(GSP_HEAP_ALIGNMENT) 95 .ok_or(EINVAL) 96 } 97 } 98 99 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 100 pub(crate) struct LibosParams { 101 /// The base amount of heap required by the GSP operating system, in bytes. 102 carveout_size: u64, 103 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 104 allowed_heap_size: Range<u64>, 105 } 106 107 impl LibosParams { 108 /// Version 2 of the GSP LIBOS (Turing and GA100) 109 const LIBOS2: LibosParams = LibosParams { 110 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 111 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 112 * u64::SZ_1M 113 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 114 }; 115 116 /// Version 3 of the GSP LIBOS (GA102+) 117 const LIBOS3: LibosParams = LibosParams { 118 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 119 allowed_heap_size: num::u32_as_u64( 120 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 121 ) * u64::SZ_1M 122 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 123 * u64::SZ_1M, 124 }; 125 126 /// Returns the libos parameters corresponding to `chipset`. 127 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 128 if chipset < Chipset::GA102 { 129 &Self::LIBOS2 130 } else { 131 &Self::LIBOS3 132 } 133 } 134 135 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 136 /// of `fb_size` (in bytes) for `chipset`. 137 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 138 // The WPR heap will contain the following: 139 // LIBOS carveout, 140 Ok(self 141 .carveout_size 142 // RM boot working memory, 143 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 144 // One RM client, 145 .saturating_add(GspFwHeapParams::client_alloc_size()) 146 // Overhead for memory management. 147 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 148 // Clamp to the supported heap sizes. 149 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 150 } 151 } 152 153 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 154 /// addresses of the GSP bootloader and firmware. 155 #[repr(transparent)] 156 pub(crate) struct GspFwWprMeta { 157 inner: bindings::GspFwWprMeta, 158 } 159 160 // SAFETY: Padding is explicit and does not contain uninitialized data. 161 unsafe impl AsBytes for GspFwWprMeta {} 162 163 // SAFETY: This struct only contains integer types for which all bit patterns 164 // are valid. 165 unsafe impl FromBytes for GspFwWprMeta {} 166 167 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 168 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 169 170 impl GspFwWprMeta { 171 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 172 /// `fb_layout` layout. 173 pub(crate) fn new<'a>( 174 gsp_firmware: &'a GspFirmware, 175 fb_layout: &'a FbLayout, 176 ) -> impl Init<Self> + 'a { 177 let init_inner = init!(bindings::GspFwWprMeta { 178 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 179 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 180 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 181 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), 182 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 183 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), 184 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 185 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 186 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 187 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 188 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 189 __bindgen_anon_1: GspFwWprMetaBootInfo { 190 sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), 191 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 192 }, 193 }, 194 gspFwRsvdStart: fb_layout.heap.start, 195 nonWprHeapOffset: fb_layout.heap.start, 196 nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, 197 gspFwWprStart: fb_layout.wpr2.start, 198 gspFwHeapOffset: fb_layout.wpr2_heap.start, 199 gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, 200 gspFwOffset: fb_layout.elf.start, 201 bootBinOffset: fb_layout.boot.start, 202 frtsOffset: fb_layout.frts.start, 203 frtsSize: fb_layout.frts.end - fb_layout.frts.start, 204 gspFwWprEnd: fb_layout 205 .vga_workspace 206 .start 207 .align_down(Alignment::new::<SZ_128K>()), 208 gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, 209 fbSize: fb_layout.fb.end - fb_layout.fb.start, 210 vgaWorkspaceOffset: fb_layout.vga_workspace.start, 211 vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, 212 pmuReservedSize: fb_layout.pmu_reserved_size, 213 ..Zeroable::init_zeroed() 214 }); 215 216 init!(GspFwWprMeta { 217 inner <- init_inner, 218 }) 219 } 220 } 221 222 #[derive(Copy, Clone, Debug, PartialEq)] 223 #[repr(u32)] 224 pub(crate) enum MsgFunction { 225 // Common function codes 226 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 227 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 228 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 229 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 230 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 231 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 232 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 233 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 234 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 235 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 236 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 237 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 238 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 239 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 240 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 241 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 242 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 243 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 244 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 245 UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, 246 247 // Event codes 248 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 249 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 250 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 251 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 252 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 253 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 254 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 255 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 256 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 257 } 258 259 impl TryFrom<u32> for MsgFunction { 260 type Error = kernel::error::Error; 261 262 fn try_from(value: u32) -> Result<MsgFunction> { 263 match value { 264 // Common function codes 265 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 266 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 267 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 268 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 269 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 270 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 271 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 272 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 273 Ok(MsgFunction::ContinuationRecord) 274 } 275 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 276 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 277 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 278 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 279 Ok(MsgFunction::GspInitPostObjGpu) 280 } 281 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 282 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 283 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 284 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 285 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 286 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 287 Ok(MsgFunction::SetGuestSystemInfo) 288 } 289 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 290 bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { 291 Ok(MsgFunction::UnloadingGuestDriver) 292 } 293 294 // Event codes 295 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 296 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 297 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 298 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 299 Ok(MsgFunction::GspRunCpuSequencer) 300 } 301 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 302 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 303 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 304 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 305 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 306 _ => Err(EINVAL), 307 } 308 } 309 } 310 311 impl From<MsgFunction> for u32 { 312 fn from(value: MsgFunction) -> Self { 313 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 314 value as u32 315 } 316 } 317 318 /// Sequencer buffer opcode for GSP sequencer commands. 319 #[derive(Copy, Clone, Debug, PartialEq)] 320 #[repr(u32)] 321 pub(crate) enum SeqBufOpcode { 322 // Core operation opcodes 323 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 324 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 325 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 326 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 327 328 // Delay opcode 329 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 330 331 // Register operation opcodes 332 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 333 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 334 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 335 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 336 } 337 338 impl TryFrom<u32> for SeqBufOpcode { 339 type Error = kernel::error::Error; 340 341 fn try_from(value: u32) -> Result<SeqBufOpcode> { 342 match value { 343 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 344 Ok(SeqBufOpcode::CoreReset) 345 } 346 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 347 Ok(SeqBufOpcode::CoreResume) 348 } 349 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 350 Ok(SeqBufOpcode::CoreStart) 351 } 352 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 353 Ok(SeqBufOpcode::CoreWaitForHalt) 354 } 355 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 356 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 357 Ok(SeqBufOpcode::RegModify) 358 } 359 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 360 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 361 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 362 _ => Err(EINVAL), 363 } 364 } 365 } 366 367 impl From<SeqBufOpcode> for u32 { 368 fn from(value: SeqBufOpcode) -> Self { 369 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 370 value as u32 371 } 372 } 373 374 /// Wrapper for GSP sequencer register write payload. 375 #[repr(transparent)] 376 #[derive(Copy, Clone, Debug)] 377 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 378 379 impl RegWritePayload { 380 /// Returns the register address. 381 pub(crate) fn addr(&self) -> u32 { 382 self.0.addr 383 } 384 385 /// Returns the value to write. 386 pub(crate) fn val(&self) -> u32 { 387 self.0.val 388 } 389 } 390 391 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 392 unsafe impl FromBytes for RegWritePayload {} 393 394 // SAFETY: Padding is explicit and will not contain uninitialized data. 395 unsafe impl AsBytes for RegWritePayload {} 396 397 /// Wrapper for GSP sequencer register modify payload. 398 #[repr(transparent)] 399 #[derive(Copy, Clone, Debug)] 400 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 401 402 impl RegModifyPayload { 403 /// Returns the register address. 404 pub(crate) fn addr(&self) -> u32 { 405 self.0.addr 406 } 407 408 /// Returns the mask to apply. 409 pub(crate) fn mask(&self) -> u32 { 410 self.0.mask 411 } 412 413 /// Returns the value to write. 414 pub(crate) fn val(&self) -> u32 { 415 self.0.val 416 } 417 } 418 419 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 420 unsafe impl FromBytes for RegModifyPayload {} 421 422 // SAFETY: Padding is explicit and will not contain uninitialized data. 423 unsafe impl AsBytes for RegModifyPayload {} 424 425 /// Wrapper for GSP sequencer register poll payload. 426 #[repr(transparent)] 427 #[derive(Copy, Clone, Debug)] 428 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 429 430 impl RegPollPayload { 431 /// Returns the register address. 432 pub(crate) fn addr(&self) -> u32 { 433 self.0.addr 434 } 435 436 /// Returns the mask to apply. 437 pub(crate) fn mask(&self) -> u32 { 438 self.0.mask 439 } 440 441 /// Returns the expected value. 442 pub(crate) fn val(&self) -> u32 { 443 self.0.val 444 } 445 446 /// Returns the timeout in microseconds. 447 pub(crate) fn timeout(&self) -> u32 { 448 self.0.timeout 449 } 450 } 451 452 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 453 unsafe impl FromBytes for RegPollPayload {} 454 455 // SAFETY: Padding is explicit and will not contain uninitialized data. 456 unsafe impl AsBytes for RegPollPayload {} 457 458 /// Wrapper for GSP sequencer delay payload. 459 #[repr(transparent)] 460 #[derive(Copy, Clone, Debug)] 461 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 462 463 impl DelayUsPayload { 464 /// Returns the delay value in microseconds. 465 pub(crate) fn val(&self) -> u32 { 466 self.0.val 467 } 468 } 469 470 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 471 unsafe impl FromBytes for DelayUsPayload {} 472 473 // SAFETY: Padding is explicit and will not contain uninitialized data. 474 unsafe impl AsBytes for DelayUsPayload {} 475 476 /// Wrapper for GSP sequencer register store payload. 477 #[repr(transparent)] 478 #[derive(Copy, Clone, Debug)] 479 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 480 481 impl RegStorePayload { 482 /// Returns the register address. 483 pub(crate) fn addr(&self) -> u32 { 484 self.0.addr 485 } 486 487 /// Returns the storage index. 488 #[allow(unused)] 489 pub(crate) fn index(&self) -> u32 { 490 self.0.index 491 } 492 } 493 494 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 495 unsafe impl FromBytes for RegStorePayload {} 496 497 // SAFETY: Padding is explicit and will not contain uninitialized data. 498 unsafe impl AsBytes for RegStorePayload {} 499 500 /// Wrapper for GSP sequencer buffer command. 501 #[repr(transparent)] 502 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 503 504 impl SequencerBufferCmd { 505 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 506 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 507 self.0.opCode.try_into() 508 } 509 510 /// Returns the register write payload by value. 511 /// 512 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 513 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 514 if self.opcode()? != SeqBufOpcode::RegWrite { 515 return Err(EINVAL); 516 } 517 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 518 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 519 } 520 521 /// Returns the register modify payload by value. 522 /// 523 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 524 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 525 if self.opcode()? != SeqBufOpcode::RegModify { 526 return Err(EINVAL); 527 } 528 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 529 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 530 } 531 532 /// Returns the register poll payload by value. 533 /// 534 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 535 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 536 if self.opcode()? != SeqBufOpcode::RegPoll { 537 return Err(EINVAL); 538 } 539 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 540 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 541 } 542 543 /// Returns the delay payload by value. 544 /// 545 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 546 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 547 if self.opcode()? != SeqBufOpcode::DelayUs { 548 return Err(EINVAL); 549 } 550 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 551 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 552 } 553 554 /// Returns the register store payload by value. 555 /// 556 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 557 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 558 if self.opcode()? != SeqBufOpcode::RegStore { 559 return Err(EINVAL); 560 } 561 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 562 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 563 } 564 } 565 566 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 567 unsafe impl FromBytes for SequencerBufferCmd {} 568 569 // SAFETY: Padding is explicit and will not contain uninitialized data. 570 unsafe impl AsBytes for SequencerBufferCmd {} 571 572 /// Wrapper for GSP run CPU sequencer RPC. 573 #[repr(transparent)] 574 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 575 576 impl RunCpuSequencer { 577 /// Returns the command index. 578 pub(crate) fn cmd_index(&self) -> u32 { 579 self.0.cmdIndex 580 } 581 } 582 583 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 584 unsafe impl FromBytes for RunCpuSequencer {} 585 586 // SAFETY: Padding is explicit and will not contain uninitialized data. 587 unsafe impl AsBytes for RunCpuSequencer {} 588 589 /// Struct containing the arguments required to pass a memory buffer to the GSP 590 /// for use during initialisation. 591 /// 592 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 593 /// configured for a larger page size (e.g. 64K pages), we need to give 594 /// the GSP an array of 4K pages. Since we only create physically contiguous 595 /// buffers the math to calculate the addresses is simple. 596 /// 597 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 598 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 599 /// buffers to be physically contiguous anyway. 600 /// 601 /// The memory allocated for the arguments must remain until the GSP sends the 602 /// init_done RPC. 603 #[repr(transparent)] 604 pub(crate) struct LibosMemoryRegionInitArgument { 605 inner: bindings::LibosMemoryRegionInitArgument, 606 } 607 608 // SAFETY: Padding is explicit and does not contain uninitialized data. 609 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 610 611 // SAFETY: This struct only contains integer types for which all bit patterns 612 // are valid. 613 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 614 615 impl LibosMemoryRegionInitArgument { 616 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 617 name: &'static str, 618 obj: &'a Coherent<A>, 619 ) -> impl Init<Self> + 'a { 620 /// Generates the `ID8` identifier required for some GSP objects. 621 fn id8(name: &str) -> u64 { 622 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 623 624 for (c, b) in name.bytes().rev().zip(&mut bytes) { 625 *b = c; 626 } 627 628 u64::from_ne_bytes(bytes) 629 } 630 631 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 632 id8: id8(name), 633 pa: obj.dma_handle(), 634 size: num::usize_as_u64(obj.size()), 635 kind: num::u32_into_u8::< 636 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 637 >(), 638 loc: num::u32_into_u8::< 639 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 640 >(), 641 ..Zeroable::init_zeroed() 642 }); 643 644 init!(LibosMemoryRegionInitArgument { 645 inner <- init_inner, 646 }) 647 } 648 } 649 650 /// TX header for setting up a message queue with the GSP. 651 #[repr(transparent)] 652 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 653 654 impl MsgqTxHeader { 655 /// Create a new TX queue header. 656 /// 657 /// # Arguments 658 /// 659 /// * `msgq_size` - Total size of the message queue structure, in bytes. 660 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 661 /// structure. 662 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 663 /// allocated for the message queue in the message queue structure. 664 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 665 Self(bindings::msgqTxHeader { 666 version: 0, 667 size: msgq_size, 668 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 669 msgCount: msg_count, 670 writePtr: 0, 671 flags: 1, 672 rxHdrOff: rx_hdr_offset, 673 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 674 }) 675 } 676 677 /// Returns the value of the write pointer for this queue. 678 pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { 679 io_read!(this, .0.writePtr) 680 } 681 682 /// Sets the value of the write pointer for this queue. 683 pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { 684 io_write!(this, .0.writePtr, val) 685 } 686 } 687 688 // SAFETY: Padding is explicit and does not contain uninitialized data. 689 unsafe impl AsBytes for MsgqTxHeader {} 690 691 /// RX header for setting up a message queue with the GSP. 692 #[repr(transparent)] 693 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 694 695 /// Header for the message RX queue. 696 impl MsgqRxHeader { 697 /// Creates a new RX queue header. 698 pub(crate) fn new() -> Self { 699 Self(Default::default()) 700 } 701 702 /// Returns the value of the read pointer for this queue. 703 pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { 704 io_read!(this, .0.readPtr) 705 } 706 707 /// Sets the value of the read pointer for this queue. 708 pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { 709 io_write!(this, .0.readPtr, val) 710 } 711 } 712 713 // SAFETY: Padding is explicit and does not contain uninitialized data. 714 unsafe impl AsBytes for MsgqRxHeader {} 715 716 bitfield! { 717 struct MsgHeaderVersion(u32) { 718 31:24 major; 719 23:16 minor; 720 } 721 } 722 723 impl MsgHeaderVersion { 724 const MAJOR_TOT: u8 = 3; 725 const MINOR_TOT: u8 = 0; 726 727 fn new() -> Self { 728 Self::zeroed() 729 .with_major(Self::MAJOR_TOT) 730 .with_minor(Self::MINOR_TOT) 731 } 732 } 733 734 impl bindings::rpc_message_header_v { 735 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 736 type RpcMessageHeader = bindings::rpc_message_header_v; 737 738 try_init!(RpcMessageHeader { 739 header_version: MsgHeaderVersion::new().into(), 740 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 741 function: function.into(), 742 length: size_of::<Self>() 743 .checked_add(cmd_size) 744 .ok_or(EOVERFLOW) 745 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 746 rpc_result: 0xffffffff, 747 rpc_result_private: 0xffffffff, 748 ..Zeroable::init_zeroed() 749 }) 750 } 751 } 752 753 /// GSP Message Element. 754 /// 755 /// This is essentially a message header expected to be followed by the message data. 756 #[repr(transparent)] 757 pub(crate) struct GspMsgElement { 758 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 759 } 760 761 impl GspMsgElement { 762 /// Creates a new message element. 763 /// 764 /// # Arguments 765 /// 766 /// * `sequence` - Sequence number of the message. 767 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 768 /// * `function` - Function of the message. 769 pub(crate) fn init( 770 sequence: u32, 771 cmd_size: usize, 772 function: MsgFunction, 773 ) -> impl Init<Self, Error> { 774 type RpcMessageHeader = bindings::rpc_message_header_v; 775 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 776 let init_inner = try_init!(InnerGspMsgElement { 777 seqNum: sequence, 778 elemCount: size_of::<Self>() 779 .checked_add(cmd_size) 780 .ok_or(EOVERFLOW)? 781 .div_ceil(GSP_PAGE_SIZE) 782 .try_into() 783 .map_err(|_| EOVERFLOW)?, 784 rpc <- RpcMessageHeader::init(cmd_size, function), 785 ..Zeroable::init_zeroed() 786 }); 787 788 try_init!(GspMsgElement { 789 inner <- init_inner, 790 }) 791 } 792 793 /// Sets the checksum of this message. 794 /// 795 /// Since the header is also part of the checksum, this is usually called after the whole 796 /// message has been written to the shared memory area. 797 pub(crate) fn set_checksum(&mut self, checksum: u32) { 798 self.inner.checkSum = checksum; 799 } 800 801 /// Returns the length of the message's payload. 802 pub(crate) fn payload_length(&self) -> usize { 803 // `rpc.length` includes the length of the RPC message header. 804 num::u32_as_usize(self.inner.rpc.length) 805 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 806 } 807 808 /// Returns the total length of the message, message and RPC headers included. 809 pub(crate) fn length(&self) -> usize { 810 size_of::<Self>() + self.payload_length() 811 } 812 813 // Returns the sequence number of the message. 814 pub(crate) fn sequence(&self) -> u32 { 815 self.inner.rpc.sequence 816 } 817 818 // Returns the function of the message, if it is valid, or the invalid function number as an 819 // error. 820 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 821 self.inner 822 .rpc 823 .function 824 .try_into() 825 .map_err(|_| self.inner.rpc.function) 826 } 827 828 // Returns the number of elements (i.e. memory pages) used by this message. 829 pub(crate) fn element_count(&self) -> u32 { 830 self.inner.elemCount 831 } 832 } 833 834 // SAFETY: Padding is explicit and does not contain uninitialized data. 835 unsafe impl AsBytes for GspMsgElement {} 836 837 // SAFETY: This struct only contains integer types for which all bit patterns 838 // are valid. 839 unsafe impl FromBytes for GspMsgElement {} 840 841 /// Arguments for GSP startup. 842 #[repr(transparent)] 843 #[derive(Zeroable)] 844 pub(crate) struct GspArgumentsCached { 845 inner: bindings::GSP_ARGUMENTS_CACHED, 846 } 847 848 impl GspArgumentsCached { 849 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 850 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 851 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 852 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 853 bDmemStack: 1, 854 ..Zeroable::init_zeroed() 855 }); 856 857 init!(GspArgumentsCached { 858 inner <- init_inner, 859 }) 860 } 861 } 862 863 // SAFETY: Padding is explicit and will not contain uninitialized data. 864 unsafe impl AsBytes for GspArgumentsCached {} 865 866 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 867 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 868 /// to that size. 869 #[repr(C)] 870 #[derive(Zeroable)] 871 pub(crate) struct GspArgumentsPadded { 872 pub(crate) inner: GspArgumentsCached, 873 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 874 } 875 876 impl GspArgumentsPadded { 877 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 878 init!(GspArgumentsPadded { 879 inner <- GspArgumentsCached::new(cmdq), 880 ..Zeroable::init_zeroed() 881 }) 882 } 883 } 884 885 // SAFETY: Padding is explicit and will not contain uninitialized data. 886 unsafe impl AsBytes for GspArgumentsPadded {} 887 888 // SAFETY: This struct only contains integer types for which all bit patterns 889 // are valid. 890 unsafe impl FromBytes for GspArgumentsPadded {} 891 892 /// Init arguments for the message queue. 893 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 894 895 impl MessageQueueInitArguments { 896 /// Creates a new init arguments structure for `cmdq`. 897 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 898 init!(MessageQueueInitArguments { 899 sharedMemPhysAddr: cmdq.dma_handle, 900 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 901 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 902 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 903 ..Zeroable::init_zeroed() 904 }) 905 } 906 } 907 908 #[repr(u32)] 909 pub(crate) enum GspDmaTarget { 910 #[expect(dead_code)] 911 LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, 912 CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, 913 NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, 914 } 915 916 type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; 917 918 impl GspAcrBootGspRmParams { 919 fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { 920 let params = init!(Self { 921 target: target as u32, 922 gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), 923 gspRmDescOffset: wpr_meta_addr, 924 bIsGspRmBoot: 1, 925 wprCarveoutOffset: 0, 926 wprCarveoutSize: 0, 927 __bindgen_padding_0: Default::default(), 928 }); 929 930 params 931 } 932 } 933 934 type GspRmParams = bindings::GSP_RM_PARAMS; 935 936 impl GspRmParams { 937 fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { 938 let params = init!(Self { 939 target: target as u32, 940 bootArgsOffset: libos_addr, 941 __bindgen_padding_0: Default::default(), 942 }); 943 944 params 945 } 946 } 947 948 pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; 949 950 // SAFETY: Padding is explicit and will not contain uninitialized data. 951 unsafe impl AsBytes for GspFmcBootParams {} 952 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 953 unsafe impl FromBytes for GspFmcBootParams {} 954 955 impl GspFmcBootParams { 956 pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { 957 let init = init!(Self { 958 // Blackwell FSP obtains WPR info from other sources, so 959 // wprCarveoutOffset and wprCarveoutSize are left zero. 960 bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, 961 wpr_meta_addr), 962 gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), 963 ..Zeroable::init_zeroed() 964 }); 965 966 init 967 } 968 } 969