1 // SPDX-License-Identifier: GPL-2.0 2 3 pub(crate) mod commands; 4 mod r570_144; 5 6 // Alias to avoid repeating the version number with every use. 7 use r570_144 as bindings; 8 9 use core::ops::Range; 10 11 use kernel::{ 12 dma::Coherent, 13 prelude::*, 14 ptr::{ 15 Alignable, 16 Alignment, 17 KnownSize, // 18 }, 19 sizes::{ 20 SizeConstants, 21 SZ_128K, // 22 }, 23 transmute::{ 24 AsBytes, 25 FromBytes, // 26 }, 27 }; 28 29 use crate::{ 30 fb::FbLayout, 31 firmware::gsp::GspFirmware, 32 gpu::Chipset, 33 gsp::{ 34 cmdq::Cmdq, // 35 GSP_PAGE_SIZE, 36 }, 37 num::{ 38 self, 39 FromSafeCast, // 40 }, 41 }; 42 43 // TODO: Replace with `IoView` projections once available. 44 pub(super) mod gsp_mem { 45 use core::sync::atomic::{ 46 fence, 47 Ordering, // 48 }; 49 50 use kernel::{ 51 dma::Coherent, 52 dma_read, 53 dma_write, // 54 }; 55 56 use crate::gsp::cmdq::{ 57 GspMem, 58 MSGQ_NUM_PAGES, // 59 }; 60 61 pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 { 62 dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES 63 } 64 65 pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 { 66 dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES 67 } 68 69 pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 { 70 dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES 71 } 72 73 pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) { 74 let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 75 76 // Ensure read pointer is properly ordered. 77 fence(Ordering::SeqCst); 78 79 dma_write!(qs, .cpuq.rx.0.readPtr, rptr); 80 } 81 82 pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 { 83 dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES 84 } 85 86 pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) { 87 let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; 88 89 dma_write!(qs, .cpuq.tx.0.writePtr, wptr); 90 91 // Ensure all command data is visible before triggering the GSP read. 92 fence(Ordering::SeqCst); 93 } 94 } 95 96 /// Maximum size of a single GSP message queue element in bytes. 97 pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = 98 num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); 99 100 /// Empty type to group methods related to heap parameters for running the GSP firmware. 101 enum GspFwHeapParams {} 102 103 /// Minimum required alignment for the GSP heap. 104 const GSP_HEAP_ALIGNMENT: Alignment = Alignment::new::<{ 1 << 20 }>(); 105 106 impl GspFwHeapParams { 107 /// Returns the amount of GSP-RM heap memory used during GSP-RM boot and initialization (up to 108 /// and including the first client subdevice allocation). 109 fn base_rm_size(_chipset: Chipset) -> u64 { 110 // TODO: this needs to be updated to return the correct value for Hopper+ once support for 111 // them is added: 112 // u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100) 113 u64::from(bindings::GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X) 114 } 115 116 /// Returns the amount of heap memory required to support a single channel allocation. 117 fn client_alloc_size() -> u64 { 118 u64::from(bindings::GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE) 119 .align_up(GSP_HEAP_ALIGNMENT) 120 .unwrap_or(u64::MAX) 121 } 122 123 /// Returns the amount of memory to reserve for management purposes for a framebuffer of size 124 /// `fb_size`. 125 fn management_overhead(fb_size: u64) -> Result<u64> { 126 let fb_size_gb = fb_size.div_ceil(u64::SZ_1G); 127 128 u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB) 129 .checked_mul(fb_size_gb) 130 .ok_or(EINVAL)? 131 .align_up(GSP_HEAP_ALIGNMENT) 132 .ok_or(EINVAL) 133 } 134 } 135 136 /// Heap memory requirements and constraints for a given version of the GSP LIBOS. 137 pub(crate) struct LibosParams { 138 /// The base amount of heap required by the GSP operating system, in bytes. 139 carveout_size: u64, 140 /// The minimum and maximum sizes allowed for the GSP FW heap, in bytes. 141 allowed_heap_size: Range<u64>, 142 } 143 144 impl LibosParams { 145 /// Version 2 of the GSP LIBOS (Turing and GA100) 146 const LIBOS2: LibosParams = LibosParams { 147 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2), 148 allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB) 149 * u64::SZ_1M 150 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M, 151 }; 152 153 /// Version 3 of the GSP LIBOS (GA102+) 154 const LIBOS3: LibosParams = LibosParams { 155 carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL), 156 allowed_heap_size: num::u32_as_u64( 157 bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB, 158 ) * u64::SZ_1M 159 ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB) 160 * u64::SZ_1M, 161 }; 162 163 /// Returns the libos parameters corresponding to `chipset`. 164 pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams { 165 if chipset < Chipset::GA102 { 166 &Self::LIBOS2 167 } else { 168 &Self::LIBOS3 169 } 170 } 171 172 /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size 173 /// of `fb_size` (in bytes) for `chipset`. 174 pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { 175 // The WPR heap will contain the following: 176 // LIBOS carveout, 177 Ok(self 178 .carveout_size 179 // RM boot working memory, 180 .saturating_add(GspFwHeapParams::base_rm_size(chipset)) 181 // One RM client, 182 .saturating_add(GspFwHeapParams::client_alloc_size()) 183 // Overhead for memory management. 184 .saturating_add(GspFwHeapParams::management_overhead(fb_size)?) 185 // Clamp to the supported heap sizes. 186 .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)) 187 } 188 } 189 190 /// Structure passed to the GSP bootloader, containing the framebuffer layout as well as the DMA 191 /// addresses of the GSP bootloader and firmware. 192 #[repr(transparent)] 193 pub(crate) struct GspFwWprMeta { 194 inner: bindings::GspFwWprMeta, 195 } 196 197 // SAFETY: Padding is explicit and does not contain uninitialized data. 198 unsafe impl AsBytes for GspFwWprMeta {} 199 200 // SAFETY: This struct only contains integer types for which all bit patterns 201 // are valid. 202 unsafe impl FromBytes for GspFwWprMeta {} 203 204 type GspFwWprMetaBootResumeInfo = bindings::GspFwWprMeta__bindgen_ty_1; 205 type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; 206 207 impl GspFwWprMeta { 208 /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the 209 /// `fb_layout` layout. 210 pub(crate) fn new<'a>( 211 gsp_firmware: &'a GspFirmware, 212 fb_layout: &'a FbLayout, 213 ) -> impl Init<Self> + 'a { 214 #[allow(non_snake_case)] 215 let init_inner = init!(bindings::GspFwWprMeta { 216 // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. 217 magic: bindings::GSP_FW_WPR_META_MAGIC as u64, 218 revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), 219 sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), 220 sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), 221 sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), 222 sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), 223 bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), 224 bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), 225 bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), 226 __bindgen_anon_1: GspFwWprMetaBootResumeInfo { 227 __bindgen_anon_1: GspFwWprMetaBootInfo { 228 sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), 229 sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), 230 }, 231 }, 232 gspFwRsvdStart: fb_layout.heap.start, 233 nonWprHeapOffset: fb_layout.heap.start, 234 nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, 235 gspFwWprStart: fb_layout.wpr2.start, 236 gspFwHeapOffset: fb_layout.wpr2_heap.start, 237 gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, 238 gspFwOffset: fb_layout.elf.start, 239 bootBinOffset: fb_layout.boot.start, 240 frtsOffset: fb_layout.frts.start, 241 frtsSize: fb_layout.frts.end - fb_layout.frts.start, 242 gspFwWprEnd: fb_layout 243 .vga_workspace 244 .start 245 .align_down(Alignment::new::<SZ_128K>()), 246 gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, 247 fbSize: fb_layout.fb.end - fb_layout.fb.start, 248 vgaWorkspaceOffset: fb_layout.vga_workspace.start, 249 vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, 250 ..Zeroable::init_zeroed() 251 }); 252 253 init!(GspFwWprMeta { 254 inner <- init_inner, 255 }) 256 } 257 } 258 259 #[derive(Copy, Clone, Debug, PartialEq)] 260 #[repr(u32)] 261 pub(crate) enum MsgFunction { 262 // Common function codes 263 AllocChannelDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA, 264 AllocCtxDma = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA, 265 AllocDevice = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE, 266 AllocMemory = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY, 267 AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT, 268 AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT, 269 BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA, 270 ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, 271 Free = bindings::NV_VGPU_MSG_FUNCTION_FREE, 272 GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, 273 GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO, 274 GspInitPostObjGpu = bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU, 275 GspRmControl = bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, 276 GspSetSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, 277 Log = bindings::NV_VGPU_MSG_FUNCTION_LOG, 278 MapMemory = bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY, 279 Nop = bindings::NV_VGPU_MSG_FUNCTION_NOP, 280 SetGuestSystemInfo = bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO, 281 SetRegistry = bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY, 282 283 // Event codes 284 GspInitDone = bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE, 285 GspLockdownNotice = bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE, 286 GspPostNoCat = bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD, 287 GspRunCpuSequencer = bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER, 288 MmuFaultQueued = bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED, 289 OsErrorLog = bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG, 290 PostEvent = bindings::NV_VGPU_MSG_EVENT_POST_EVENT, 291 RcTriggered = bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED, 292 UcodeLibOsPrint = bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, 293 } 294 295 impl TryFrom<u32> for MsgFunction { 296 type Error = kernel::error::Error; 297 298 fn try_from(value: u32) -> Result<MsgFunction> { 299 match value { 300 // Common function codes 301 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CHANNEL_DMA => Ok(MsgFunction::AllocChannelDma), 302 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_CTX_DMA => Ok(MsgFunction::AllocCtxDma), 303 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_DEVICE => Ok(MsgFunction::AllocDevice), 304 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_MEMORY => Ok(MsgFunction::AllocMemory), 305 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject), 306 bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot), 307 bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma), 308 bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => { 309 Ok(MsgFunction::ContinuationRecord) 310 } 311 bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free), 312 bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo), 313 bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo), 314 bindings::NV_VGPU_MSG_FUNCTION_GSP_INIT_POST_OBJGPU => { 315 Ok(MsgFunction::GspInitPostObjGpu) 316 } 317 bindings::NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL => Ok(MsgFunction::GspRmControl), 318 bindings::NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO => Ok(MsgFunction::GspSetSystemInfo), 319 bindings::NV_VGPU_MSG_FUNCTION_LOG => Ok(MsgFunction::Log), 320 bindings::NV_VGPU_MSG_FUNCTION_MAP_MEMORY => Ok(MsgFunction::MapMemory), 321 bindings::NV_VGPU_MSG_FUNCTION_NOP => Ok(MsgFunction::Nop), 322 bindings::NV_VGPU_MSG_FUNCTION_SET_GUEST_SYSTEM_INFO => { 323 Ok(MsgFunction::SetGuestSystemInfo) 324 } 325 bindings::NV_VGPU_MSG_FUNCTION_SET_REGISTRY => Ok(MsgFunction::SetRegistry), 326 327 // Event codes 328 bindings::NV_VGPU_MSG_EVENT_GSP_INIT_DONE => Ok(MsgFunction::GspInitDone), 329 bindings::NV_VGPU_MSG_EVENT_GSP_LOCKDOWN_NOTICE => Ok(MsgFunction::GspLockdownNotice), 330 bindings::NV_VGPU_MSG_EVENT_GSP_POST_NOCAT_RECORD => Ok(MsgFunction::GspPostNoCat), 331 bindings::NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER => { 332 Ok(MsgFunction::GspRunCpuSequencer) 333 } 334 bindings::NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED => Ok(MsgFunction::MmuFaultQueued), 335 bindings::NV_VGPU_MSG_EVENT_OS_ERROR_LOG => Ok(MsgFunction::OsErrorLog), 336 bindings::NV_VGPU_MSG_EVENT_POST_EVENT => Ok(MsgFunction::PostEvent), 337 bindings::NV_VGPU_MSG_EVENT_RC_TRIGGERED => Ok(MsgFunction::RcTriggered), 338 bindings::NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT => Ok(MsgFunction::UcodeLibOsPrint), 339 _ => Err(EINVAL), 340 } 341 } 342 } 343 344 impl From<MsgFunction> for u32 { 345 fn from(value: MsgFunction) -> Self { 346 // CAST: `MsgFunction` is `repr(u32)` and can thus be cast losslessly. 347 value as u32 348 } 349 } 350 351 /// Sequencer buffer opcode for GSP sequencer commands. 352 #[derive(Copy, Clone, Debug, PartialEq)] 353 #[repr(u32)] 354 pub(crate) enum SeqBufOpcode { 355 // Core operation opcodes 356 CoreReset = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET, 357 CoreResume = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME, 358 CoreStart = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START, 359 CoreWaitForHalt = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT, 360 361 // Delay opcode 362 DelayUs = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US, 363 364 // Register operation opcodes 365 RegModify = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY, 366 RegPoll = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL, 367 RegStore = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE, 368 RegWrite = bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE, 369 } 370 371 impl TryFrom<u32> for SeqBufOpcode { 372 type Error = kernel::error::Error; 373 374 fn try_from(value: u32) -> Result<SeqBufOpcode> { 375 match value { 376 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESET => { 377 Ok(SeqBufOpcode::CoreReset) 378 } 379 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_RESUME => { 380 Ok(SeqBufOpcode::CoreResume) 381 } 382 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_START => { 383 Ok(SeqBufOpcode::CoreStart) 384 } 385 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT => { 386 Ok(SeqBufOpcode::CoreWaitForHalt) 387 } 388 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_DELAY_US => Ok(SeqBufOpcode::DelayUs), 389 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_MODIFY => { 390 Ok(SeqBufOpcode::RegModify) 391 } 392 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_POLL => Ok(SeqBufOpcode::RegPoll), 393 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_STORE => Ok(SeqBufOpcode::RegStore), 394 bindings::GSP_SEQ_BUF_OPCODE_GSP_SEQ_BUF_OPCODE_REG_WRITE => Ok(SeqBufOpcode::RegWrite), 395 _ => Err(EINVAL), 396 } 397 } 398 } 399 400 impl From<SeqBufOpcode> for u32 { 401 fn from(value: SeqBufOpcode) -> Self { 402 // CAST: `SeqBufOpcode` is `repr(u32)` and can thus be cast losslessly. 403 value as u32 404 } 405 } 406 407 /// Wrapper for GSP sequencer register write payload. 408 #[repr(transparent)] 409 #[derive(Copy, Clone, Debug)] 410 pub(crate) struct RegWritePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_WRITE); 411 412 impl RegWritePayload { 413 /// Returns the register address. 414 pub(crate) fn addr(&self) -> u32 { 415 self.0.addr 416 } 417 418 /// Returns the value to write. 419 pub(crate) fn val(&self) -> u32 { 420 self.0.val 421 } 422 } 423 424 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 425 unsafe impl FromBytes for RegWritePayload {} 426 427 // SAFETY: Padding is explicit and will not contain uninitialized data. 428 unsafe impl AsBytes for RegWritePayload {} 429 430 /// Wrapper for GSP sequencer register modify payload. 431 #[repr(transparent)] 432 #[derive(Copy, Clone, Debug)] 433 pub(crate) struct RegModifyPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_MODIFY); 434 435 impl RegModifyPayload { 436 /// Returns the register address. 437 pub(crate) fn addr(&self) -> u32 { 438 self.0.addr 439 } 440 441 /// Returns the mask to apply. 442 pub(crate) fn mask(&self) -> u32 { 443 self.0.mask 444 } 445 446 /// Returns the value to write. 447 pub(crate) fn val(&self) -> u32 { 448 self.0.val 449 } 450 } 451 452 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 453 unsafe impl FromBytes for RegModifyPayload {} 454 455 // SAFETY: Padding is explicit and will not contain uninitialized data. 456 unsafe impl AsBytes for RegModifyPayload {} 457 458 /// Wrapper for GSP sequencer register poll payload. 459 #[repr(transparent)] 460 #[derive(Copy, Clone, Debug)] 461 pub(crate) struct RegPollPayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_POLL); 462 463 impl RegPollPayload { 464 /// Returns the register address. 465 pub(crate) fn addr(&self) -> u32 { 466 self.0.addr 467 } 468 469 /// Returns the mask to apply. 470 pub(crate) fn mask(&self) -> u32 { 471 self.0.mask 472 } 473 474 /// Returns the expected value. 475 pub(crate) fn val(&self) -> u32 { 476 self.0.val 477 } 478 479 /// Returns the timeout in microseconds. 480 pub(crate) fn timeout(&self) -> u32 { 481 self.0.timeout 482 } 483 } 484 485 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 486 unsafe impl FromBytes for RegPollPayload {} 487 488 // SAFETY: Padding is explicit and will not contain uninitialized data. 489 unsafe impl AsBytes for RegPollPayload {} 490 491 /// Wrapper for GSP sequencer delay payload. 492 #[repr(transparent)] 493 #[derive(Copy, Clone, Debug)] 494 pub(crate) struct DelayUsPayload(bindings::GSP_SEQ_BUF_PAYLOAD_DELAY_US); 495 496 impl DelayUsPayload { 497 /// Returns the delay value in microseconds. 498 pub(crate) fn val(&self) -> u32 { 499 self.0.val 500 } 501 } 502 503 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 504 unsafe impl FromBytes for DelayUsPayload {} 505 506 // SAFETY: Padding is explicit and will not contain uninitialized data. 507 unsafe impl AsBytes for DelayUsPayload {} 508 509 /// Wrapper for GSP sequencer register store payload. 510 #[repr(transparent)] 511 #[derive(Copy, Clone, Debug)] 512 pub(crate) struct RegStorePayload(bindings::GSP_SEQ_BUF_PAYLOAD_REG_STORE); 513 514 impl RegStorePayload { 515 /// Returns the register address. 516 pub(crate) fn addr(&self) -> u32 { 517 self.0.addr 518 } 519 520 /// Returns the storage index. 521 #[allow(unused)] 522 pub(crate) fn index(&self) -> u32 { 523 self.0.index 524 } 525 } 526 527 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 528 unsafe impl FromBytes for RegStorePayload {} 529 530 // SAFETY: Padding is explicit and will not contain uninitialized data. 531 unsafe impl AsBytes for RegStorePayload {} 532 533 /// Wrapper for GSP sequencer buffer command. 534 #[repr(transparent)] 535 pub(crate) struct SequencerBufferCmd(bindings::GSP_SEQUENCER_BUFFER_CMD); 536 537 impl SequencerBufferCmd { 538 /// Returns the opcode as a `SeqBufOpcode` enum, or error if invalid. 539 pub(crate) fn opcode(&self) -> Result<SeqBufOpcode> { 540 self.0.opCode.try_into() 541 } 542 543 /// Returns the register write payload by value. 544 /// 545 /// Returns an error if the opcode is not `SeqBufOpcode::RegWrite`. 546 pub(crate) fn reg_write_payload(&self) -> Result<RegWritePayload> { 547 if self.opcode()? != SeqBufOpcode::RegWrite { 548 return Err(EINVAL); 549 } 550 // SAFETY: Opcode is verified to be `RegWrite`, so union contains valid `RegWritePayload`. 551 Ok(RegWritePayload(unsafe { self.0.payload.regWrite })) 552 } 553 554 /// Returns the register modify payload by value. 555 /// 556 /// Returns an error if the opcode is not `SeqBufOpcode::RegModify`. 557 pub(crate) fn reg_modify_payload(&self) -> Result<RegModifyPayload> { 558 if self.opcode()? != SeqBufOpcode::RegModify { 559 return Err(EINVAL); 560 } 561 // SAFETY: Opcode is verified to be `RegModify`, so union contains valid `RegModifyPayload`. 562 Ok(RegModifyPayload(unsafe { self.0.payload.regModify })) 563 } 564 565 /// Returns the register poll payload by value. 566 /// 567 /// Returns an error if the opcode is not `SeqBufOpcode::RegPoll`. 568 pub(crate) fn reg_poll_payload(&self) -> Result<RegPollPayload> { 569 if self.opcode()? != SeqBufOpcode::RegPoll { 570 return Err(EINVAL); 571 } 572 // SAFETY: Opcode is verified to be `RegPoll`, so union contains valid `RegPollPayload`. 573 Ok(RegPollPayload(unsafe { self.0.payload.regPoll })) 574 } 575 576 /// Returns the delay payload by value. 577 /// 578 /// Returns an error if the opcode is not `SeqBufOpcode::DelayUs`. 579 pub(crate) fn delay_us_payload(&self) -> Result<DelayUsPayload> { 580 if self.opcode()? != SeqBufOpcode::DelayUs { 581 return Err(EINVAL); 582 } 583 // SAFETY: Opcode is verified to be `DelayUs`, so union contains valid `DelayUsPayload`. 584 Ok(DelayUsPayload(unsafe { self.0.payload.delayUs })) 585 } 586 587 /// Returns the register store payload by value. 588 /// 589 /// Returns an error if the opcode is not `SeqBufOpcode::RegStore`. 590 pub(crate) fn reg_store_payload(&self) -> Result<RegStorePayload> { 591 if self.opcode()? != SeqBufOpcode::RegStore { 592 return Err(EINVAL); 593 } 594 // SAFETY: Opcode is verified to be `RegStore`, so union contains valid `RegStorePayload`. 595 Ok(RegStorePayload(unsafe { self.0.payload.regStore })) 596 } 597 } 598 599 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 600 unsafe impl FromBytes for SequencerBufferCmd {} 601 602 // SAFETY: Padding is explicit and will not contain uninitialized data. 603 unsafe impl AsBytes for SequencerBufferCmd {} 604 605 /// Wrapper for GSP run CPU sequencer RPC. 606 #[repr(transparent)] 607 pub(crate) struct RunCpuSequencer(bindings::rpc_run_cpu_sequencer_v17_00); 608 609 impl RunCpuSequencer { 610 /// Returns the command index. 611 pub(crate) fn cmd_index(&self) -> u32 { 612 self.0.cmdIndex 613 } 614 } 615 616 // SAFETY: This struct only contains integer types for which all bit patterns are valid. 617 unsafe impl FromBytes for RunCpuSequencer {} 618 619 // SAFETY: Padding is explicit and will not contain uninitialized data. 620 unsafe impl AsBytes for RunCpuSequencer {} 621 622 /// Struct containing the arguments required to pass a memory buffer to the GSP 623 /// for use during initialisation. 624 /// 625 /// The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is 626 /// configured for a larger page size (e.g. 64K pages), we need to give 627 /// the GSP an array of 4K pages. Since we only create physically contiguous 628 /// buffers the math to calculate the addresses is simple. 629 /// 630 /// The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently 631 /// ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the 632 /// buffers to be physically contiguous anyway. 633 /// 634 /// The memory allocated for the arguments must remain until the GSP sends the 635 /// init_done RPC. 636 #[repr(transparent)] 637 pub(crate) struct LibosMemoryRegionInitArgument { 638 inner: bindings::LibosMemoryRegionInitArgument, 639 } 640 641 // SAFETY: Padding is explicit and does not contain uninitialized data. 642 unsafe impl AsBytes for LibosMemoryRegionInitArgument {} 643 644 // SAFETY: This struct only contains integer types for which all bit patterns 645 // are valid. 646 unsafe impl FromBytes for LibosMemoryRegionInitArgument {} 647 648 impl LibosMemoryRegionInitArgument { 649 pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( 650 name: &'static str, 651 obj: &'a Coherent<A>, 652 ) -> impl Init<Self> + 'a { 653 /// Generates the `ID8` identifier required for some GSP objects. 654 fn id8(name: &str) -> u64 { 655 let mut bytes = [0u8; core::mem::size_of::<u64>()]; 656 657 for (c, b) in name.bytes().rev().zip(&mut bytes) { 658 *b = c; 659 } 660 661 u64::from_ne_bytes(bytes) 662 } 663 664 #[allow(non_snake_case)] 665 let init_inner = init!(bindings::LibosMemoryRegionInitArgument { 666 id8: id8(name), 667 pa: obj.dma_handle(), 668 size: num::usize_as_u64(obj.size()), 669 kind: num::u32_into_u8::< 670 { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, 671 >(), 672 loc: num::u32_into_u8::< 673 { bindings::LibosMemoryRegionLoc_LIBOS_MEMORY_REGION_LOC_SYSMEM }, 674 >(), 675 ..Zeroable::init_zeroed() 676 }); 677 678 init!(LibosMemoryRegionInitArgument { 679 inner <- init_inner, 680 }) 681 } 682 } 683 684 /// TX header for setting up a message queue with the GSP. 685 #[repr(transparent)] 686 pub(crate) struct MsgqTxHeader(bindings::msgqTxHeader); 687 688 impl MsgqTxHeader { 689 /// Create a new TX queue header. 690 /// 691 /// # Arguments 692 /// 693 /// * `msgq_size` - Total size of the message queue structure, in bytes. 694 /// * `rx_hdr_offset` - Offset, in bytes, of the start of the RX header in the message queue 695 /// structure. 696 /// * `msg_count` - Number of messages that can be sent, i.e. the number of memory pages 697 /// allocated for the message queue in the message queue structure. 698 pub(crate) fn new(msgq_size: u32, rx_hdr_offset: u32, msg_count: u32) -> Self { 699 Self(bindings::msgqTxHeader { 700 version: 0, 701 size: msgq_size, 702 msgSize: num::usize_into_u32::<GSP_PAGE_SIZE>(), 703 msgCount: msg_count, 704 writePtr: 0, 705 flags: 1, 706 rxHdrOff: rx_hdr_offset, 707 entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), 708 }) 709 } 710 } 711 712 // SAFETY: Padding is explicit and does not contain uninitialized data. 713 unsafe impl AsBytes for MsgqTxHeader {} 714 715 /// RX header for setting up a message queue with the GSP. 716 #[repr(transparent)] 717 pub(crate) struct MsgqRxHeader(bindings::msgqRxHeader); 718 719 /// Header for the message RX queue. 720 impl MsgqRxHeader { 721 /// Creates a new RX queue header. 722 pub(crate) fn new() -> Self { 723 Self(Default::default()) 724 } 725 } 726 727 // SAFETY: Padding is explicit and does not contain uninitialized data. 728 unsafe impl AsBytes for MsgqRxHeader {} 729 730 bitfield! { 731 struct MsgHeaderVersion(u32) { 732 31:24 major as u8; 733 23:16 minor as u8; 734 } 735 } 736 737 impl MsgHeaderVersion { 738 const MAJOR_TOT: u8 = 3; 739 const MINOR_TOT: u8 = 0; 740 741 fn new() -> Self { 742 Self::default() 743 .set_major(Self::MAJOR_TOT) 744 .set_minor(Self::MINOR_TOT) 745 } 746 } 747 748 impl bindings::rpc_message_header_v { 749 fn init(cmd_size: usize, function: MsgFunction) -> impl Init<Self, Error> { 750 type RpcMessageHeader = bindings::rpc_message_header_v; 751 752 try_init!(RpcMessageHeader { 753 header_version: MsgHeaderVersion::new().into(), 754 signature: bindings::NV_VGPU_MSG_SIGNATURE_VALID, 755 function: function.into(), 756 length: size_of::<Self>() 757 .checked_add(cmd_size) 758 .ok_or(EOVERFLOW) 759 .and_then(|v| v.try_into().map_err(|_| EINVAL))?, 760 rpc_result: 0xffffffff, 761 rpc_result_private: 0xffffffff, 762 ..Zeroable::init_zeroed() 763 }) 764 } 765 } 766 767 /// GSP Message Element. 768 /// 769 /// This is essentially a message header expected to be followed by the message data. 770 #[repr(transparent)] 771 pub(crate) struct GspMsgElement { 772 inner: bindings::GSP_MSG_QUEUE_ELEMENT, 773 } 774 775 impl GspMsgElement { 776 /// Creates a new message element. 777 /// 778 /// # Arguments 779 /// 780 /// * `sequence` - Sequence number of the message. 781 /// * `cmd_size` - Size of the command (not including the message element), in bytes. 782 /// * `function` - Function of the message. 783 #[allow(non_snake_case)] 784 pub(crate) fn init( 785 sequence: u32, 786 cmd_size: usize, 787 function: MsgFunction, 788 ) -> impl Init<Self, Error> { 789 type RpcMessageHeader = bindings::rpc_message_header_v; 790 type InnerGspMsgElement = bindings::GSP_MSG_QUEUE_ELEMENT; 791 let init_inner = try_init!(InnerGspMsgElement { 792 seqNum: sequence, 793 elemCount: size_of::<Self>() 794 .checked_add(cmd_size) 795 .ok_or(EOVERFLOW)? 796 .div_ceil(GSP_PAGE_SIZE) 797 .try_into() 798 .map_err(|_| EOVERFLOW)?, 799 rpc <- RpcMessageHeader::init(cmd_size, function), 800 ..Zeroable::init_zeroed() 801 }); 802 803 try_init!(GspMsgElement { 804 inner <- init_inner, 805 }) 806 } 807 808 /// Sets the checksum of this message. 809 /// 810 /// Since the header is also part of the checksum, this is usually called after the whole 811 /// message has been written to the shared memory area. 812 pub(crate) fn set_checksum(&mut self, checksum: u32) { 813 self.inner.checkSum = checksum; 814 } 815 816 /// Returns the length of the message's payload. 817 pub(crate) fn payload_length(&self) -> usize { 818 // `rpc.length` includes the length of the RPC message header. 819 num::u32_as_usize(self.inner.rpc.length) 820 .saturating_sub(size_of::<bindings::rpc_message_header_v>()) 821 } 822 823 /// Returns the total length of the message, message and RPC headers included. 824 pub(crate) fn length(&self) -> usize { 825 size_of::<Self>() + self.payload_length() 826 } 827 828 // Returns the sequence number of the message. 829 pub(crate) fn sequence(&self) -> u32 { 830 self.inner.rpc.sequence 831 } 832 833 // Returns the function of the message, if it is valid, or the invalid function number as an 834 // error. 835 pub(crate) fn function(&self) -> Result<MsgFunction, u32> { 836 self.inner 837 .rpc 838 .function 839 .try_into() 840 .map_err(|_| self.inner.rpc.function) 841 } 842 843 // Returns the number of elements (i.e. memory pages) used by this message. 844 pub(crate) fn element_count(&self) -> u32 { 845 self.inner.elemCount 846 } 847 } 848 849 // SAFETY: Padding is explicit and does not contain uninitialized data. 850 unsafe impl AsBytes for GspMsgElement {} 851 852 // SAFETY: This struct only contains integer types for which all bit patterns 853 // are valid. 854 unsafe impl FromBytes for GspMsgElement {} 855 856 /// Arguments for GSP startup. 857 #[repr(transparent)] 858 #[derive(Zeroable)] 859 pub(crate) struct GspArgumentsCached { 860 inner: bindings::GSP_ARGUMENTS_CACHED, 861 } 862 863 impl GspArgumentsCached { 864 /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. 865 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 866 #[allow(non_snake_case)] 867 let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { 868 messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), 869 bDmemStack: 1, 870 ..Zeroable::init_zeroed() 871 }); 872 873 init!(GspArgumentsCached { 874 inner <- init_inner, 875 }) 876 } 877 } 878 879 // SAFETY: Padding is explicit and will not contain uninitialized data. 880 unsafe impl AsBytes for GspArgumentsCached {} 881 882 /// On Turing and GA100, the entries in the `LibosMemoryRegionInitArgument` 883 /// must all be a multiple of GSP_PAGE_SIZE in size, so add padding to force it 884 /// to that size. 885 #[repr(C)] 886 #[derive(Zeroable)] 887 pub(crate) struct GspArgumentsPadded { 888 pub(crate) inner: GspArgumentsCached, 889 _padding: [u8; GSP_PAGE_SIZE - core::mem::size_of::<bindings::GSP_ARGUMENTS_CACHED>()], 890 } 891 892 impl GspArgumentsPadded { 893 pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 894 init!(GspArgumentsPadded { 895 inner <- GspArgumentsCached::new(cmdq), 896 ..Zeroable::init_zeroed() 897 }) 898 } 899 } 900 901 // SAFETY: Padding is explicit and will not contain uninitialized data. 902 unsafe impl AsBytes for GspArgumentsPadded {} 903 904 // SAFETY: This struct only contains integer types for which all bit patterns 905 // are valid. 906 unsafe impl FromBytes for GspArgumentsPadded {} 907 908 /// Init arguments for the message queue. 909 type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; 910 911 impl MessageQueueInitArguments { 912 /// Creates a new init arguments structure for `cmdq`. 913 #[allow(non_snake_case)] 914 fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { 915 init!(MessageQueueInitArguments { 916 sharedMemPhysAddr: cmdq.dma_handle, 917 pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), 918 cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), 919 statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), 920 ..Zeroable::init_zeroed() 921 }) 922 } 923 } 924