1 // SPDX-License-Identifier: GPL-2.0 2 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 use kernel::{ 5 io::{ 6 register, 7 Io, 8 Mmio, // 9 }, 10 sizes::SizeConstants, 11 time, // 12 }; 13 use pin_init::Zeroable; 14 15 use crate::{ 16 driver::NovaRegisters, 17 falcon::{ 18 DmaTrfCmdSize, 19 FalconCoreRev, 20 FalconCoreRevSubversion, 21 FalconFbifMemType, 22 FalconFbifTarget, 23 FalconMem, 24 FalconModSelAlgo, 25 FalconSecurityModel, 26 PFalcon2Registers, 27 PFalconRegisters, 28 PeregrineCoreSelect, // 29 }, 30 mm::tlb::TlbAckMode, // 31 }; 32 33 // PBUS 34 35 register! { 36 base: NovaRegisters; 37 38 pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} 39 } 40 41 // PGC6 register space. 42 // 43 // `GC6` is a GPU low-power state where VRAM is in self-refresh and the GPU is powered down (except 44 // for power rails needed to keep self-refresh working and important registers and hardware 45 // blocks). 46 // 47 // These scratch registers remain powered on even in a low-power state and have a designated group 48 // number. 49 50 register! { 51 base: NovaRegisters; 52 53 /// Boot Sequence Interface (BSI) register used to determine 54 /// if GSP reload/resume has completed during the boot process. 55 pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 { 56 26:26 boot_stage_3_handoff => bool; 57 } 58 59 /// Privilege level mask register. It dictates whether the host CPU has privilege to access the 60 /// `PGC6_AON_SECURE_SCRATCH_GROUP_05` register (which it needs to read GFW_BOOT). 61 pub(crate) NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK(u32) @ 0x00118128 { 62 /// Set after FWSEC lowers its protection level. 63 0:0 read_protection_level0 => bool; 64 } 65 66 /// OpenRM defines this as a register array, but doesn't specify its size and only uses its 67 /// first element. Be conservative until we know the actual size or need to use more registers. 68 pub(crate) NV_PGC6_AON_SECURE_SCRATCH_GROUP_05(u32)[1] @ 0x00118234 {} 69 70 /// Scratch group 05 register 0 used as GFW boot progress indicator. 71 pub(crate) NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT(u32) 72 => NV_PGC6_AON_SECURE_SCRATCH_GROUP_05[0] { 73 /// Progress of GFW boot (0xff means completed). 74 7:0 progress; 75 } 76 77 pub(crate) NV_PGC6_AON_SECURE_SCRATCH_GROUP_42(u32) @ 0x001183a4 { 78 31:0 value; 79 } 80 81 /// Scratch group 42 register used as framebuffer size. 82 pub(crate) NV_USABLE_FB_SIZE_IN_MB(u32) => NV_PGC6_AON_SECURE_SCRATCH_GROUP_42 { 83 /// Usable framebuffer size, in megabytes. 84 31:0 value; 85 } 86 } 87 88 impl NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT { 89 /// Returns `true` if GFW boot is completed. 90 pub(crate) fn completed(self) -> bool { 91 self.progress() == 0xff 92 } 93 } 94 95 impl NV_USABLE_FB_SIZE_IN_MB { 96 /// Returns the usable framebuffer size, in bytes. 97 pub(crate) fn usable_fb_size(self) -> u64 { 98 u64::from(self.value()) * u64::SZ_1M 99 } 100 } 101 102 // FUSE 103 104 pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; 105 106 register! { 107 base: NovaRegisters; 108 109 pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 { 110 15:0 data => u16; 111 } 112 113 pub(crate) NV_FUSE_OPT_FPF_SEC2_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824140 { 114 15:0 data => u16; 115 } 116 117 pub(crate) NV_FUSE_OPT_FPF_GSP_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x008241c0 { 118 15:0 data => u16; 119 } 120 } 121 122 // PFALCON 123 124 register! { 125 base: PFalconRegisters; 126 127 /// Clears the latch of every cause whose bit is written as `1`. Write-only. 128 /// 129 /// The write ends the latch and not the source, so a cause driven from outside the falcon 130 /// stays set. "Retriggering a falcon" in `Documentation/gpu/nova/core/interrupts.rst` names 131 /// those causes. 132 pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 { 133 6:6 swgen0 => bool; 134 4:4 halt => bool; 135 } 136 137 /// Interrupt causes latched in the falcon, one bit per cause, whichever target each is routed 138 /// to. 139 /// 140 /// The causes routed to the host are the ones also set in `NV_PRISCV_RISCV_IRQMASK` and 141 /// `NV_PRISCV_RISCV_IRQDEST`. 142 pub(crate) NV_PFALCON_FALCON_IRQSTAT(u32) @ 0x00000008 { 143 6:6 swgen0 => bool; 144 } 145 146 pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 { 147 31:0 value => u32; 148 } 149 150 pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 { 151 31:0 value => u32; 152 } 153 154 /// Used to store version information about the firmware running 155 /// on the Falcon processor. 156 pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 { 157 31:0 value => u32; 158 } 159 160 pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 { 161 31:0 value => u32; 162 } 163 164 pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 { 165 /// Signal indicating that reset is completed (GA102+). 166 31:31 reset_ready => bool; 167 /// RISC-V branch privilege lockdown bit. 168 13:13 riscv_br_priv_lockdown => bool; 169 /// Set to 0 after memory scrubbing is completed. 170 12:12 mem_scrubbing => bool; 171 10:10 riscv => bool; 172 } 173 174 pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 { 175 6:6 alias_en => bool; 176 4:4 halted => bool; 177 1:1 startcpu => bool; 178 } 179 180 pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 { 181 31:0 value => u32; 182 } 183 184 pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c { 185 7:7 secure_stat => bool; 186 6:3 dmaq_num; 187 2:2 imem_scrubbing => bool; 188 1:1 dmem_scrubbing => bool; 189 0:0 require_ctx => bool; 190 } 191 192 pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 { 193 31:0 base => u32; 194 } 195 196 pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 { 197 23:0 offs; 198 } 199 200 pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 { 201 16:16 set_dmtag; 202 14:12 ctxdma; 203 10:8 size ?=> DmaTrfCmdSize; 204 5:5 is_write => bool; 205 4:4 imem => bool; 206 3:2 sec; 207 1:1 idle => bool; 208 0:0 full => bool; 209 } 210 211 pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c { 212 31:0 offs => u32; 213 } 214 215 pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 { 216 8:0 base; 217 } 218 219 pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c { 220 /// Core revision subversion. 221 7:6 core_rev_subversion => FalconCoreRevSubversion; 222 /// Security model. 223 5:4 security_model ?=> FalconSecurityModel; 224 /// Core revision. 225 3:0 core_rev ?=> FalconCoreRev; 226 } 227 228 pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 { 229 1:1 startcpu => bool; 230 } 231 232 /// IMEM access control register. Up to 4 ports are available for IMEM access. 233 pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 { 234 /// Access secure IMEM. 235 28:28 secure => bool; 236 /// Auto-increment on write. 237 24:24 aincw => bool; 238 /// IMEM block and word offset. 239 15:0 offs; 240 } 241 242 /// IMEM data register. Reading/writing this register accesses IMEM at the address 243 /// specified by the corresponding IMEMC register. 244 pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 { 245 31:0 data; 246 } 247 248 /// IMEM tag register. Used to set the tag for the current IMEM block. 249 pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 { 250 15:0 tag; 251 } 252 253 /// DMEM access control register. Up to 8 ports are available for DMEM access. 254 pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 { 255 /// Auto-increment on write. 256 24:24 aincw => bool; 257 /// DMEM block and word offset. 258 15:0 offs; 259 } 260 261 /// DMEM data register. Reading/writing this register accesses DMEM at the address 262 /// specified by the corresponding DMEMC register. 263 pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 { 264 31:0 data; 265 } 266 267 /// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the 268 /// falcon instance. 269 pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 { 270 0:0 reset => bool; 271 } 272 273 /// Makes the falcon re-emit its host-routed causes into the interrupt tree. Write-only. 274 /// 275 /// Present from GA100 on. See "Retriggering a falcon" in 276 /// `Documentation/gpu/nova/core/interrupts.rst`. 277 /// 278 /// The hardware headers declare two elements, and OpenRM writes only the first. 279 pub(crate) NV_PFALCON_FALCON_INTR_RETRIGGER(u32)[2] @ 0x000003e8 { 280 0:0 trigger => bool; 281 } 282 283 pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 { 284 2:2 mem_type => FalconFbifMemType; 285 1:0 target ?=> FalconFbifTarget; 286 } 287 288 pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 { 289 7:7 allow_phys_no_ctx => bool; 290 } 291 292 // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). 293 // These provide the falcon external memory communication interface. 294 295 pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 { 296 /// EMEM byte offset (4-byte aligned) within the block. 297 7:2 offs; 298 /// EMEM block to access. 299 15:8 blk; 300 /// Auto-increment the offset after each write. 301 24:24 aincw => bool; 302 /// Auto-increment the offset after each read. 303 25:25 aincr => bool; 304 } 305 306 pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 { 307 31:0 data => u32; 308 } 309 } 310 311 impl NV_PFALCON_FALCON_DMACTL { 312 /// Returns `true` if memory scrubbing is completed. 313 pub(crate) fn mem_scrubbing_done(self) -> bool { 314 !self.dmem_scrubbing() && !self.imem_scrubbing() 315 } 316 } 317 318 impl NV_PFALCON_FALCON_DMATRFCMD { 319 /// Programs the `imem` and `sec` fields for the given FalconMem 320 pub(crate) fn with_falcon_mem(self, mem: FalconMem) -> Self { 321 let this = self.with_imem(mem != FalconMem::Dmem); 322 323 match mem { 324 FalconMem::ImemSecure => this.with_const_sec::<1>(), 325 _ => this.with_const_sec::<0>(), 326 } 327 } 328 } 329 330 impl NV_PFALCON_FALCON_ENGINE { 331 /// Resets the falcon 332 pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) { 333 pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true)); 334 335 // TIMEOUT: falcon engine should not take more than 10us to reset. 336 time::delay::fsleep(time::Delta::from_micros(10)); 337 338 pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false)); 339 } 340 } 341 342 impl NV_PFALCON_FALCON_HWCFG2 { 343 /// Returns `true` if memory scrubbing is completed. 344 pub(crate) fn mem_scrubbing_done(self) -> bool { 345 !self.mem_scrubbing() 346 } 347 } 348 349 /* PFALCON2 */ 350 351 register! { 352 base: PFalcon2Registers; 353 354 pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 { 355 7:0 algo ?=> FalconModSelAlgo; 356 } 357 358 pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 { 359 7:0 ucode_id => u8; 360 } 361 362 pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c { 363 31:0 value => u32; 364 } 365 366 /// OpenRM defines this as a register array, but doesn't specify its size and only uses its 367 /// first element. Be conservative until we know the actual size or need to use more registers. 368 pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 { 369 31:0 value => u32; 370 } 371 } 372 373 // PRISCV 374 375 register! { 376 base: PFalcon2Registers; 377 378 /// RISC-V status register for debug (Turing and GA100 only). 379 /// Reflects current RISC-V core status. 380 pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 { 381 /// RISC-V core active/inactive status. 382 0:0 active_stat => bool; 383 } 384 385 /// GA102 and later. 386 pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 { 387 7:7 active_stat => bool; 388 4:4 halted => bool; 389 } 390 391 /// GA102 and later. 392 pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 { 393 8:8 br_fetch => bool; 394 4:4 core_select => PeregrineCoreSelect; 395 0:0 valid => bool; 396 } 397 } 398 399 // FSP (Foundation Security Processor) queue registers for Hopper/Blackwell Chain of Trust. 400 // These registers manage falcon EMEM communication queues. 401 402 register! { 403 base: NovaRegisters; 404 405 pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { 406 31:0 address => u32; 407 } 408 409 pub(crate) NV_PFSP_QUEUE_TAIL(u32)[8] @ 0x008f2c04 { 410 31:0 address => u32; 411 } 412 413 pub(crate) NV_PFSP_MSGQ_HEAD(u32)[8] @ 0x008f2c80 { 414 31:0 val => u32; 415 } 416 417 pub(crate) NV_PFSP_MSGQ_TAIL(u32)[8] @ 0x008f2c84 { 418 31:0 val => u32; 419 } 420 } 421 422 // The modules below provide registers that are not identical on all supported chips. They should 423 // only be used in HAL modules. 424 425 pub(crate) mod gm107 { 426 use kernel::io::register; 427 428 use crate::driver::NovaRegisters; 429 430 // FUSE 431 432 register! { 433 base: NovaRegisters; 434 435 pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 { 436 0:0 display_disabled => bool; 437 } 438 } 439 } 440 441 pub(crate) mod tu102 { 442 use kernel::io::register; 443 444 use crate::falcon::PFalcon2Registers; 445 446 // The RISC-V interrupt routing registers, at the offsets that Turing and GA100 use. 447 448 register! { 449 base: PFalcon2Registers; 450 451 /// Enabled causes, one bit per cause. Read-only to the host. 452 pub(crate) NV_PRISCV_RISCV_IRQMASK(u32) @ 0x000002b4 { 453 31:0 value => u32; 454 } 455 456 /// Causes routed to the host, one bit per cause. A clear bit routes the cause to the 457 /// RISC-V core. 458 pub(crate) NV_PRISCV_RISCV_IRQDEST(u32) @ 0x000002b8 { 459 31:0 value => u32; 460 } 461 } 462 } 463 464 pub(crate) mod ga100 { 465 use kernel::io::register; 466 467 use crate::driver::NovaRegisters; 468 469 // FUSE 470 471 register! { 472 base: NovaRegisters; 473 474 pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 { 475 0:0 display_disabled => bool; 476 } 477 } 478 } 479 480 pub(crate) mod ga102 { 481 use kernel::io::register; 482 483 use crate::falcon::PFalcon2Registers; 484 485 // The RISC-V interrupt routing registers, at the offsets that GA102 and later use. 486 487 register! { 488 base: PFalcon2Registers; 489 490 /// Same as [`super::tu102::NV_PRISCV_RISCV_IRQMASK`], at the GA102 offset. 491 pub(crate) NV_PRISCV_RISCV_IRQMASK(u32) @ 0x00000528 { 492 31:0 value => u32; 493 } 494 495 /// Same as [`super::tu102::NV_PRISCV_RISCV_IRQDEST`], at the GA102 offset. 496 pub(crate) NV_PRISCV_RISCV_IRQDEST(u32) @ 0x0000052c { 497 31:0 value => u32; 498 } 499 } 500 } 501 502 pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0xff; 503 504 pub(crate) mod gh100 { 505 use kernel::io::register; 506 507 use crate::driver::NovaRegisters; 508 509 // PTHERM 510 511 register! { 512 base: NovaRegisters; 513 514 pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { 515 31:0 data; 516 } 517 518 // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. 519 pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { 520 31:0 fsp_boot_complete; 521 } 522 } 523 } 524 525 pub(crate) mod gb202 { 526 use kernel::io::register; 527 528 use crate::driver::NovaRegisters; 529 530 // PTHERM 531 532 register! { 533 base: NovaRegisters; 534 535 pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { 536 31:0 data; 537 } 538 539 // Alias to `NV_THERM_I2CS_SCRATCH` when used to check for FSP boot completion. 540 pub(crate) NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE(u32) => NV_THERM_I2CS_SCRATCH { 541 31:0 fsp_boot_complete; 542 } 543 } 544 } 545 546 // MMU TLB 547 548 register! { 549 base: NovaRegisters; 550 551 /// TLB flush register: PDB address lower bits. 552 pub(crate) NV_TLB_FLUSH_PDB_LO(u32) @ 0x00b830a0 { 553 /// PDB address bits [39:8]. 554 31:0 pdb_lo => u32; 555 } 556 557 /// TLB flush register: PDB address higher bits. 558 pub(crate) NV_TLB_FLUSH_PDB_HI(u32) @ 0x00b830a4 { 559 /// PDB address bits [47:40]. 560 7:0 pdb_hi => u8; 561 } 562 563 /// TLB flush control register. 564 pub(crate) NV_TLB_FLUSH_CTRL(u32) @ 0x00b830b0 { 565 /// Invalidate every VA in the PDB selected by `NV_TLB_FLUSH_PDB_LO/HI`. 566 0:0 all_va => bool; 567 /// Invalidate TLBs for all PDBs (ignores `NV_TLB_FLUSH_PDB_LO/HI`). 568 1:1 all_pdb => bool; 569 /// Restrict the flush to the HUB MMU's TLBs; skip broadcasting to the 570 /// per-GPC L2 TLBs. 571 /// 572 /// The GPU MMU has a two-level TLB hierarchy: 573 /// 1. The *HUB MMU* sits at the top and serves memory requests from 574 /// "host-side" engines: the host/channel interface, copy engines, 575 /// display, and BAR1/BAR2 accesses. 576 /// 2. Each GPC (Graphics Processing Cluster — the block that houses 577 /// shader cores / SMs) has its own L2 TLB that serves requests from 578 /// the compute and graphics engines inside the cluster. 579 /// 580 /// When set, only the HUB TLBs are invalidated. This is a performance 581 /// optimization for flushes that only affect HUB-side mappings (e.g. 582 /// BAR1/BAR2 windows), where fanning the invalidation out to every 583 /// GPC's L2 TLB would be wasted work. Must be false when flushing 584 /// mappings that may be cached by compute/graphics engines. 585 2:2 hubtlb_only => bool; 586 /// Invalidation acknowledgment scope. See [`TlbAckMode`] for details. 587 8:7 ack ?=> TlbAckMode; 588 /// Write 1 to kick off the flush. Hardware clears this bit when the 589 /// flush completes; reads as 1 while the flush is in progress. 590 31:31 trigger => bool; 591 } 592 } 593 594 impl NV_TLB_FLUSH_PDB_LO { 595 /// Create a register value from a PDB address. 596 /// 597 /// Extracts bits [39:8] of the address and shifts it right by 8 bits. 598 pub(crate) fn from_pdb_addr(addr: u64) -> Self { 599 Self::zeroed().with_pdb_lo(((addr >> 8) & 0xFFFF_FFFF) as u32) 600 } 601 } 602 603 impl NV_TLB_FLUSH_PDB_HI { 604 /// Create a register value from a PDB address. 605 /// 606 /// Extracts bits [47:40] of the address and shifts it right by 40 bits. 607 pub(crate) fn from_pdb_addr(addr: u64) -> Self { 608 Self::zeroed().with_pdb_hi(((addr >> 40) & 0xFF) as u8) 609 } 610 } 611