1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 pub(crate) mod commands; 5 mod r570_144; 6 7 // Alias to avoid repeating the version number with every use. 8 use r570_144 as bindings; 9 10 use core::ops::Range; 11 12 use kernel::{ 13 bitfield, 14 dma::{ 15 Coherent, 16 CoherentView, // 17 }, 18 io::{ 19 io_read, 20 io_write, // 21 }, 22 prelude::*, 23 ptr::{ 24 Alignable, 25 Alignment, 26 KnownSize, // 27 }, 28 sizes::{ 29 SizeConstants, 30 SZ_128K, // 31 }, 32 transmute::{ 33 AsBytes, 34 FromBytes, // 35 }, 36 }; 37 38 use crate::{ 39 fb::{ 40 FbRanges, 41 FbSizes, // 42 }, 43 firmware::gsp::GspFirmware, 44 gpu::{ 45 Architecture, 46 Chipset, // 47 }, 48 gsp::{ 49 cmdq::Cmdq, // 50 GSP_PAGE_SIZE, 51 }, 52 num::{ 53 self, 54 FromSafeCast, // 55 }, 56 }; 57 58 /// Maximum size of a single GSP message queue element in bytes. 59 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 60 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 61 62 /// Empty type to group methods related to heap parameters for running the GSP firmware. 63 enum GspFwHeapParams {} 64 65 /// Minimum required alignment for the GSP heap. 66 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 67 68 impl GspFwHeapParams { 69 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 70 /// and including the first client subdevice allocation). 71 fn base_rm_size(chipset: Chipset) -> u64 { 72 match chipset.arch() { 73 Architecture::Turing | Architecture::Ampere | Architecture::Ada => { 74 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 75 } 76 Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { 77 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 78 } 79 } 80 } 81 82 /// Returns the amount of heap memory required to support a single channel allocation. 83 fn client_alloc_size() -> u64 { 84 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 85 .align_up(GSP_HEAP_ALIGNMENT) 86 .unwrap_or(u64::MAX) 87 } 88 89 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 90 /// `fb_size`. 91 fn management_overhead(fb_size: u64) -> Result<u64> { 92 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 93 94 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 95 .checked_mul(fb_size_gb) 96 .ok_or(EINVAL)? 97 .align_up(GSP_HEAP_ALIGNMENT) 98 .ok_or(EINVAL) 99 } 100 } 101 102 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 103 pub(crate) struct LibosParams { 104 /// The base amount of heap required by the GSP operating system, in bytes. 105 carveout_size: u64, 106 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 107 allowed_heap_size: Range<u64>, 108 } 109 110 impl LibosParams { 111 /// Version 2 of the GSP LIBOS (Turing and GA100) 112 const LIBOS2: LibosParams = LibosParams { 113 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 114 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 115 * u64::SZ_1M 116 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 117 }; 118 119 /// Version 3 of the GSP LIBOS (GA102+) 120 const LIBOS3: LibosParams = LibosParams { 121 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 122 allowed_heap_size: num::u32_as_u64( 123 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 124 ) * u64::SZ_1M 125 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 126 * u64::SZ_1M, 127 }; 128 129 /// Returns the libos parameters corresponding to `chipset`. 130 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 131 if chipset < Chipset::GA102 { 132 &Self::LIBOS2 133 } else { 134 &Self::LIBOS3 135 } 136 } 137 138 /// Returns the WPR heap size to reserve when vGPU is enabled. 139 pub(crate) fn vgpu_wpr_heap_size() -> u64 { 140 u64::from(bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT) 141 } 142 143 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 144 /// of `fb_size` (in bytes) for `chipset`. 145 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 146 // The WPR heap will contain the following: 147 // LIBOS carveout, 148 Ok(self 149 .carveout_size 150 // RM boot working memory, 151 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 152 // One RM client, 153 .saturating_add(GspFwHeapParams::client_alloc_size()) 154 // Overhead for memory management. 155 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 156 // Clamp to the supported heap sizes. 157 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 158 } 159 } 160 161 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 162 /// addresses of the GSP bootloader and firmware. 163 #[repr(transparent)] 164 pub(crate) struct GspFwWprMeta { 165 inner: bindings::GspFwWprMeta, 166 } 167 168 // SAFETY: Padding is explicit and does not contain uninitialized data. 169 unsafe impl AsBytes for GspFwWprMeta {} 170 171 // SAFETY: This struct only contains integer types for which all bit patterns 172 // are valid. 173 unsafe impl FromBytes for GspFwWprMeta {} 174 175 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 176 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 177 178 impl GspFwWprMeta { 179 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 180 /// framebuffer ranges `ranges`. 181 pub(crate) fn from_ranges<'a>( 182 gsp_firmware: &'a GspFirmware, 183 ranges: &'a FbRanges, 184 ) -> impl Init<Self> + 'a { 185 let init_inner = init!(bindings::GspFwWprMeta { 186 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 187 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 188 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 189 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), 190 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 191 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), 192 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 193 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 194 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 195 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 196 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 197 __bindgen_anon_1: GspFwWprMetaBootInfo { 198 sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), 199 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 200 }, 201 }, 202 gspFwRsvdStart: ranges.non_wpr_heap.start, 203 nonWprHeapOffset: ranges.non_wpr_heap.start, 204 nonWprHeapSize: ranges.non_wpr_heap.len(), 205 gspFwWprStart: ranges.wpr2.start, 206 gspFwHeapOffset: ranges.wpr2_heap.start, 207 gspFwHeapSize: ranges.wpr2_heap.len(), 208 gspFwOffset: ranges.elf.start, 209 bootBinOffset: ranges.boot.start, 210 frtsOffset: ranges.frts.start, 211 frtsSize: ranges.frts.len(), 212 gspFwWprEnd: ranges 213 .vga_workspace 214 .start 215 .align_down(Alignment::new::<SZ_128K>()), 216 gspFwHeapVfPartitionCount: ranges.vf_partition_count, 217 fbSize: ranges.fb.len(), 218 vgaWorkspaceOffset: ranges.vga_workspace.start, 219 vgaWorkspaceSize: ranges.vga_workspace.len(), 220 pmuReservedSize: ranges.pmu_reserved_size, 221 ..Zeroable::init_zeroed() 222 }); 223 224 init!(GspFwWprMeta { 225 inner <- init_inner, 226 }) 227 } 228 229 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 230 /// framebuffer region sizes `sizes`. 231 /// 232 /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. 233 pub(crate) fn from_sizes<'a>( 234 gsp_firmware: &'a GspFirmware, 235 sizes: &'a FbSizes, 236 ) -> impl Init<Self> + 'a { 237 /// VGA workspace size to reserve at the end of the framebuffer, in bytes. 238 const VGA_WORKSPACE_SIZE: u64 = u64::SZ_128K; 239 240 let init_inner = init!(bindings::GspFwWprMeta { 241 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 242 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 243 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 244 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), 245 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 246 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), 247 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 248 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 249 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 250 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 251 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 252 __bindgen_anon_1: GspFwWprMetaBootInfo { 253 sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), 254 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 255 }, 256 }, 257 nonWprHeapSize: sizes.non_wpr_heap_size, 258 gspFwHeapSize: sizes.wpr2_heap_size, 259 frtsSize: sizes.frts_size, 260 gspFwHeapVfPartitionCount: sizes.vf_partition_count, 261 vgaWorkspaceSize: VGA_WORKSPACE_SIZE, 262 pmuReservedSize: sizes.pmu_reserved_size, 263 ..Zeroable::init_zeroed() 264 }); 265 266 init!(GspFwWprMeta { 267 inner <- init_inner, 268 }) 269 } 270 } 271 272 #[derive(Copy, Clone, Debug, PartialEq)] 273 #[repr(u32)] 274 pub(crate) enum MsgFunction { 275 // Common function codes 276 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 277 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 278 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 279 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 280 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 281 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 282 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 283 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 284 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 285 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 286 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 287 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 288 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 289 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 290 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 291 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 292 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 293 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 294 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 295 UnloadingGuestDriver = bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, 296 297 // Event codes 298 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 299 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 300 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 301 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 302 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 303 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 304 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 305 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 306 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 307 } 308 309 impl TryFrom<u32> for MsgFunction { 310 type Error = kernel::error::Error; 311 312 fn try_from(value: u32) -> Result<MsgFunction> { 313 match value { 314 // Common function codes 315 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 316 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 317 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 318 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 319 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 320 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 321 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 322 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 323 Ok(MsgFunction::ContinuationRecord) 324 } 325 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 326 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 327 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 328 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 329 Ok(MsgFunction::GspInitPostObjGpu) 330 } 331 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 332 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 333 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 334 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 335 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 336 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 337 Ok(MsgFunction::SetGuestSystemInfo) 338 } 339 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 340 bindings::NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER => { 341 Ok(MsgFunction::UnloadingGuestDriver) 342 } 343 344 // Event codes 345 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 346 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 347 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 348 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 349 Ok(MsgFunction::GspRunCpuSequencer) 350 } 351 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 352 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 353 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 354 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 355 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 356 _ => Err(EINVAL), 357 } 358 } 359 } 360 361 impl From<MsgFunction> for u32 { 362 fn from(value: MsgFunction) -> Self { 363 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 364 value as u32 365 } 366 } 367 368 /// Sequencer buffer opcode for GSP sequencer commands. 369 #[derive(Copy, Clone, Debug, PartialEq)] 370 #[repr(u32)] 371 pub(crate) enum SeqBufOpcode { 372 // Core operation opcodes 373 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 374 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 375 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 376 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 377 378 // Delay opcode 379 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 380 381 // Register operation opcodes 382 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 383 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 384 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 385 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 386 } 387 388 impl TryFrom<u32> for SeqBufOpcode { 389 type Error = kernel::error::Error; 390 391 fn try_from(value: u32) -> Result<SeqBufOpcode> { 392 match value { 393 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 394 Ok(SeqBufOpcode::CoreReset) 395 } 396 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 397 Ok(SeqBufOpcode::CoreResume) 398 } 399 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 400 Ok(SeqBufOpcode::CoreStart) 401 } 402 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 403 Ok(SeqBufOpcode::CoreWaitForHalt) 404 } 405 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 406 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 407 Ok(SeqBufOpcode::RegModify) 408 } 409 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 410 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 411 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 412 _ => Err(EINVAL), 413 } 414 } 415 } 416 417 impl From<SeqBufOpcode> for u32 { 418 fn from(value: SeqBufOpcode) -> Self { 419 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 420 value as u32 421 } 422 } 423 424 /// Wrapper for GSP sequencer register write payload. 425 #[repr(transparent)] 426 #[derive(Copy, Clone, Debug)] 427 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 428 429 impl RegWritePayload { 430 /// Returns the register address. 431 pub(crate) fn addr(&self) -> u32 { 432 self.0.addr 433 } 434 435 /// Returns the value to write. 436 pub(crate) fn val(&self) -> u32 { 437 self.0.val 438 } 439 } 440 441 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 442 unsafe impl FromBytes for RegWritePayload {} 443 444 // SAFETY: Padding is explicit and will not contain uninitialized data. 445 unsafe impl AsBytes for RegWritePayload {} 446 447 /// Wrapper for GSP sequencer register modify payload. 448 #[repr(transparent)] 449 #[derive(Copy, Clone, Debug)] 450 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 451 452 impl RegModifyPayload { 453 /// Returns the register address. 454 pub(crate) fn addr(&self) -> u32 { 455 self.0.addr 456 } 457 458 /// Returns the mask to apply. 459 pub(crate) fn mask(&self) -> u32 { 460 self.0.mask 461 } 462 463 /// Returns the value to write. 464 pub(crate) fn val(&self) -> u32 { 465 self.0.val 466 } 467 } 468 469 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 470 unsafe impl FromBytes for RegModifyPayload {} 471 472 // SAFETY: Padding is explicit and will not contain uninitialized data. 473 unsafe impl AsBytes for RegModifyPayload {} 474 475 /// Wrapper for GSP sequencer register poll payload. 476 #[repr(transparent)] 477 #[derive(Copy, Clone, Debug)] 478 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 479 480 impl RegPollPayload { 481 /// Returns the register address. 482 pub(crate) fn addr(&self) -> u32 { 483 self.0.addr 484 } 485 486 /// Returns the mask to apply. 487 pub(crate) fn mask(&self) -> u32 { 488 self.0.mask 489 } 490 491 /// Returns the expected value. 492 pub(crate) fn val(&self) -> u32 { 493 self.0.val 494 } 495 496 /// Returns the timeout in microseconds. 497 pub(crate) fn timeout(&self) -> u32 { 498 self.0.timeout 499 } 500 } 501 502 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 503 unsafe impl FromBytes for RegPollPayload {} 504 505 // SAFETY: Padding is explicit and will not contain uninitialized data. 506 unsafe impl AsBytes for RegPollPayload {} 507 508 /// Wrapper for GSP sequencer delay payload. 509 #[repr(transparent)] 510 #[derive(Copy, Clone, Debug)] 511 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 512 513 impl DelayUsPayload { 514 /// Returns the delay value in microseconds. 515 pub(crate) fn val(&self) -> u32 { 516 self.0.val 517 } 518 } 519 520 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 521 unsafe impl FromBytes for DelayUsPayload {} 522 523 // SAFETY: Padding is explicit and will not contain uninitialized data. 524 unsafe impl AsBytes for DelayUsPayload {} 525 526 /// Wrapper for GSP sequencer register store payload. 527 #[repr(transparent)] 528 #[derive(Copy, Clone, Debug)] 529 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 530 531 impl RegStorePayload { 532 /// Returns the register address. 533 pub(crate) fn addr(&self) -> u32 { 534 self.0.addr 535 } 536 537 /// Returns the storage index. 538 #[allow(unused)] 539 pub(crate) fn index(&self) -> u32 { 540 self.0.index 541 } 542 } 543 544 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 545 unsafe impl FromBytes for RegStorePayload {} 546 547 // SAFETY: Padding is explicit and will not contain uninitialized data. 548 unsafe impl AsBytes for RegStorePayload {} 549 550 /// Wrapper for GSP sequencer buffer command. 551 #[repr(transparent)] 552 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 553 554 impl SequencerBufferCmd { 555 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 556 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 557 self.0.opCode.try_into() 558 } 559 560 /// Returns the register write payload by value. 561 /// 562 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 563 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 564 if self.opcode()? != SeqBufOpcode::RegWrite { 565 return Err(EINVAL); 566 } 567 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 568 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 569 } 570 571 /// Returns the register modify payload by value. 572 /// 573 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 574 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 575 if self.opcode()? != SeqBufOpcode::RegModify { 576 return Err(EINVAL); 577 } 578 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 579 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 580 } 581 582 /// Returns the register poll payload by value. 583 /// 584 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 585 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 586 if self.opcode()? != SeqBufOpcode::RegPoll { 587 return Err(EINVAL); 588 } 589 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 590 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 591 } 592 593 /// Returns the delay payload by value. 594 /// 595 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 596 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 597 if self.opcode()? != SeqBufOpcode::DelayUs { 598 return Err(EINVAL); 599 } 600 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 601 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 602 } 603 604 /// Returns the register store payload by value. 605 /// 606 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 607 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 608 if self.opcode()? != SeqBufOpcode::RegStore { 609 return Err(EINVAL); 610 } 611 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 612 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 613 } 614 } 615 616 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 617 unsafe impl FromBytes for SequencerBufferCmd {} 618 619 // SAFETY: Padding is explicit and will not contain uninitialized data. 620 unsafe impl AsBytes for SequencerBufferCmd {} 621 622 /// Wrapper for GSP run CPU sequencer RPC. 623 #[repr(transparent)] 624 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 625 626 impl RunCpuSequencer { 627 /// Returns the command index. 628 pub(crate) fn cmd_index(&self) -> u32 { 629 self.0.cmdIndex 630 } 631 } 632 633 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 634 unsafe impl FromBytes for RunCpuSequencer {} 635 636 // SAFETY: Padding is explicit and will not contain uninitialized data. 637 unsafe impl AsBytes for RunCpuSequencer {} 638 639 /// Struct containing the arguments required to pass a memory buffer to the GSP 640 /// for use during initialisation. 641 /// 642 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 643 /// configured for a larger page size (e.g. 64K pages), we need to give 644 /// the GSP an array of 4K pages. Since we only create physically contiguous 645 /// buffers the math to calculate the addresses is simple. 646 /// 647 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 648 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 649 /// buffers to be physically contiguous anyway. 650 /// 651 /// The memory allocated for the arguments must remain until the GSP sends the 652 /// init_done RPC. 653 #[repr(transparent)] 654 pub(crate) struct LibosMemoryRegionInitArgument { 655 inner: bindings::LibosMemoryRegionInitArgument, 656 } 657 658 // SAFETY: Padding is explicit and does not contain uninitialized data. 659 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 660 661 // SAFETY: This struct only contains integer types for which all bit patterns 662 // are valid. 663 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 664 665 impl LibosMemoryRegionInitArgument { 666 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 667 name: &'static str, 668 obj: &'a Coherent<A>, 669 ) -> impl Init<Self> + 'a { 670 /// Generates the `ID8` identifier required for some GSP objects. 671 fn id8(name: &str) -> u64 { 672 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 673 674 for (c, b) in name.bytes().rev().zip(&mut bytes) { 675 *b = c; 676 } 677 678 u64::from_ne_bytes(bytes) 679 } 680 681 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 682 id8: id8(name), 683 pa: obj.dma_address(), 684 size: num::usize_as_u64(obj.size()), 685 kind: num::u32_into_u8::< 686 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 687 >(), 688 loc: num::u32_into_u8::< 689 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 690 >(), 691 ..Zeroable::init_zeroed() 692 }); 693 694 init!(LibosMemoryRegionInitArgument { 695 inner <- init_inner, 696 }) 697 } 698 } 699 700 /// TX header for setting up a message queue with the GSP. 701 #[repr(transparent)] 702 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 703 704 impl MsgqTxHeader { 705 /// Create a new TX queue header. 706 /// 707 /// # Arguments 708 /// 709 /// * `msgq_size` - Total size of the message queue structure, in bytes. 710 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 711 /// structure. 712 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 713 /// allocated for the message queue in the message queue structure. 714 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 715 Self(bindings::msgqTxHeader { 716 version: 0, 717 size: msgq_size, 718 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 719 msgCount: msg_count, 720 writePtr: 0, 721 flags: 1, 722 rxHdrOff: rx_hdr_offset, 723 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 724 }) 725 } 726 727 /// Returns the value of the write pointer for this queue. 728 pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { 729 io_read!(this, .0.writePtr) 730 } 731 732 /// Sets the value of the write pointer for this queue. 733 pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { 734 io_write!(this, .0.writePtr, val) 735 } 736 } 737 738 // SAFETY: Padding is explicit and does not contain uninitialized data. 739 unsafe impl AsBytes for MsgqTxHeader {} 740 741 /// RX header for setting up a message queue with the GSP. 742 #[repr(transparent)] 743 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 744 745 /// Header for the message RX queue. 746 impl MsgqRxHeader { 747 /// Creates a new RX queue header. 748 pub(crate) fn new() -> Self { 749 Self(Default::default()) 750 } 751 752 /// Returns the value of the read pointer for this queue. 753 pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { 754 io_read!(this, .0.readPtr) 755 } 756 757 /// Sets the value of the read pointer for this queue. 758 pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { 759 io_write!(this, .0.readPtr, val) 760 } 761 } 762 763 // SAFETY: Padding is explicit and does not contain uninitialized data. 764 unsafe impl AsBytes for MsgqRxHeader {} 765 766 bitfield! { 767 struct MsgHeaderVersion(u32) { 768 31:24 major; 769 23:16 minor; 770 } 771 } 772 773 impl MsgHeaderVersion { 774 const MAJOR_TOT: u8 = 3; 775 const MINOR_TOT: u8 = 0; 776 777 fn new() -> Self { 778 Self::zeroed() 779 .with_major(Self::MAJOR_TOT) 780 .with_minor(Self::MINOR_TOT) 781 } 782 } 783 784 impl bindings::rpc_message_header_v { 785 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 786 type RpcMessageHeader = bindings::rpc_message_header_v; 787 788 try_init!(RpcMessageHeader { 789 header_version: MsgHeaderVersion::new().into(), 790 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 791 function: function.into(), 792 length: size_of::<Self>() 793 .checked_add(cmd_size) 794 .ok_or(EOVERFLOW) 795 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 796 rpc_result: 0xffffffff, 797 rpc_result_private: 0xffffffff, 798 ..Zeroable::init_zeroed() 799 }) 800 } 801 } 802 803 /// GSP Message Element. 804 /// 805 /// This is essentially a message header expected to be followed by the message data. 806 #[repr(transparent)] 807 pub(crate) struct GspMsgElement { 808 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 809 } 810 811 impl GspMsgElement { 812 /// Creates a new message element. 813 /// 814 /// # Arguments 815 /// 816 /// * `sequence` - Sequence number of the message. 817 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 818 /// * `function` - Function of the message. 819 pub(crate) fn init( 820 sequence: u32, 821 cmd_size: usize, 822 function: MsgFunction, 823 ) -> impl Init<Self, Error> { 824 type RpcMessageHeader = bindings::rpc_message_header_v; 825 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 826 let init_inner = try_init!(InnerGspMsgElement { 827 seqNum: sequence, 828 elemCount: size_of::<Self>() 829 .checked_add(cmd_size) 830 .ok_or(EOVERFLOW)? 831 .div_ceil(GSP_PAGE_SIZE) 832 .try_into() 833 .map_err(|_| EOVERFLOW)?, 834 rpc <- RpcMessageHeader::init(cmd_size, function), 835 ..Zeroable::init_zeroed() 836 }); 837 838 try_init!(GspMsgElement { 839 inner <- init_inner, 840 }) 841 } 842 843 /// Sets the checksum of this message. 844 /// 845 /// Since the header is also part of the checksum, this is usually called after the whole 846 /// message has been written to the shared memory area. 847 pub(crate) fn set_checksum(&mut self, checksum: u32) { 848 self.inner.checkSum = checksum; 849 } 850 851 /// Returns the length of the message's payload. 852 pub(crate) fn payload_length(&self) -> usize { 853 // `rpc.length` includes the length of the RPC message header. 854 num::u32_as_usize(self.inner.rpc.length) 855 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 856 } 857 858 /// Returns the total length of the message, message and RPC headers included. 859 pub(crate) fn length(&self) -> usize { 860 size_of::<Self>() + self.payload_length() 861 } 862 863 // Returns the sequence number of the message. 864 pub(crate) fn sequence(&self) -> u32 { 865 self.inner.rpc.sequence 866 } 867 868 // Returns the function of the message, if it is valid, or the invalid function number as an 869 // error. 870 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 871 self.inner 872 .rpc 873 .function 874 .try_into() 875 .map_err(|_| self.inner.rpc.function) 876 } 877 878 // Returns the number of elements (i.e. memory pages) used by this message. 879 pub(crate) fn element_count(&self) -> u32 { 880 self.inner.elemCount 881 } 882 } 883 884 // SAFETY: Padding is explicit and does not contain uninitialized data. 885 unsafe impl AsBytes for GspMsgElement {} 886 887 // SAFETY: This struct only contains integer types for which all bit patterns 888 // are valid. 889 unsafe impl FromBytes for GspMsgElement {} 890 891 /// Arguments for GSP startup. 892 #[repr(transparent)] 893 #[derive(Zeroable)] 894 pub(crate) struct GspArgumentsCached { 895 inner: bindings::GSP_ARGUMENTS_CACHED, 896 } 897 898 impl GspArgumentsCached { 899 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 900 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 901 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 902 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 903 bDmemStack: 1, 904 ..Zeroable::init_zeroed() 905 }); 906 907 init!(GspArgumentsCached { 908 inner <- init_inner, 909 }) 910 } 911 } 912 913 // SAFETY: Padding is explicit and will not contain uninitialized data. 914 unsafe impl AsBytes for GspArgumentsCached {} 915 916 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 917 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 918 /// to that size. 919 #[repr(C)] 920 #[derive(Zeroable)] 921 pub(crate) struct GspArgumentsPadded { 922 pub(crate) inner: GspArgumentsCached, 923 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 924 } 925 926 impl GspArgumentsPadded { 927 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 928 init!(GspArgumentsPadded { 929 inner <- GspArgumentsCached::new(cmdq), 930 ..Zeroable::init_zeroed() 931 }) 932 } 933 } 934 935 // SAFETY: Padding is explicit and will not contain uninitialized data. 936 unsafe impl AsBytes for GspArgumentsPadded {} 937 938 // SAFETY: This struct only contains integer types for which all bit patterns 939 // are valid. 940 unsafe impl FromBytes for GspArgumentsPadded {} 941 942 /// Init arguments for the message queue. 943 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 944 945 impl MessageQueueInitArguments { 946 /// Creates a new init arguments structure for `cmdq`. 947 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 948 init!(MessageQueueInitArguments { 949 sharedMemPhysAddr: cmdq.dma_addr, 950 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 951 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 952 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 953 ..Zeroable::init_zeroed() 954 }) 955 } 956 } 957 958 #[repr(u32)] 959 pub(crate) enum GspDmaTarget { 960 #[expect(dead_code)] 961 LocalFb = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_LOCAL_FB, 962 CoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_COHERENT_SYSTEM, 963 NoncoherentSystem = bindings::GSP_DMA_TARGET_GSP_DMA_TARGET_NONCOHERENT_SYSTEM, 964 } 965 966 type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; 967 968 impl GspAcrBootGspRmParams { 969 fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { 970 let params = init!(Self { 971 target: target as u32, 972 gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), 973 gspRmDescOffset: wpr_meta_addr, 974 bIsGspRmBoot: 1, 975 wprCarveoutOffset: 0, 976 wprCarveoutSize: 0, 977 __bindgen_padding_0: Default::default(), 978 }); 979 980 params 981 } 982 } 983 984 type GspRmParams = bindings::GSP_RM_PARAMS; 985 986 impl GspRmParams { 987 fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { 988 let params = init!(Self { 989 target: target as u32, 990 bootArgsOffset: libos_addr, 991 __bindgen_padding_0: Default::default(), 992 }); 993 994 params 995 } 996 } 997 998 pub(crate) type GspFmcBootParams = bindings::GSP_FMC_BOOT_PARAMS; 999 1000 // SAFETY: Padding is explicit and will not contain uninitialized data. 1001 unsafe impl AsBytes for GspFmcBootParams {} 1002 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 1003 unsafe impl FromBytes for GspFmcBootParams {} 1004 1005 impl GspFmcBootParams { 1006 pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { 1007 let init = init!(Self { 1008 // Blackwell FSP obtains WPR info from other sources, so 1009 // wprCarveoutOffset and wprCarveoutSize are left zero. 1010 bootGspRmParams <- GspAcrBootGspRmParams::new(GspDmaTarget::CoherentSystem, 1011 wpr_meta_addr), 1012 gspRmParams <- GspRmParams::new(GspDmaTarget::NoncoherentSystem, libos_addr), 1013 ..Zeroable::init_zeroed() 1014 }); 1015 1016 init 1017 } 1018 } 1019