1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 3 //! # Definitions 4 //! 5 //! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) 6 //! from the command stream. 7 //! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular 8 //! job or sequence of jobs. The instructions exist in one or more command buffers. 9 //! - **CSF**: Command Stream Frontend - The interface and implementation for job submission 10 //! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS 11 //! interfaces. 12 //! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple 13 //! CSGs, and each CSG contains multiple CSs. 14 //! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the 15 //! iterator control aspects. Implements the CSF in conjunction with the MCU. 16 //! - **GLB**: Global - Prefix for global interface registers that control operations common to 17 //! all CSs. 18 //! - **JASID**: Job Address Space ID - Identifies the address space for a job. 19 //! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream 20 //! hardware. 21 //! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. 22 23 // We don't expect that all the registers and fields will be used, even in the 24 // future. 25 // 26 // Nevertheless, it is useful to have most of them defined, like the C driver 27 // does. 28 #![expect(dead_code)] 29 30 /// Combine two 32-bit values into a single 64-bit value. 31 pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { 32 (u64::from(lo)) | ((u64::from(hi)) << 32) 33 } 34 35 /// Read a logical 64-bit value from split 32-bit registers without tearing. 36 pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { 37 loop { 38 let hi1 = hi_read(); 39 let lo = lo_read(); 40 let hi2 = hi_read(); 41 42 if hi1 == hi2 { 43 return join_u64(lo, hi1); 44 } 45 } 46 } 47 48 pub(crate) use mmu_control::mmu_as_control::MAX_AS; 49 50 /// These registers correspond to the GPU_CONTROL register page. 51 /// They are involved in GPU configuration and control. 52 pub(crate) mod gpu_control { 53 use kernel::{ 54 num::Bounded, 55 prelude::*, 56 register, 57 uapi, // 58 }; 59 60 register! { 61 /// GPU identification register. 62 pub(crate) GPU_ID(u32) @ 0x0 { 63 /// Status of the GPU release. 64 3:0 ver_status; 65 /// Minor release version number. 66 11:4 ver_minor; 67 /// Major release version number. 68 15:12 ver_major; 69 /// Product identifier. 70 19:16 prod_major; 71 /// Architecture patch revision. 72 23:20 arch_rev; 73 /// Architecture minor revision. 74 27:24 arch_minor; 75 /// Architecture major revision. 76 31:28 arch_major; 77 } 78 79 /// Level 2 cache features register. 80 pub(crate) L2_FEATURES(u32) @ 0x4 { 81 /// Cache line size. 82 7:0 line_size; 83 /// Cache associativity. 84 15:8 associativity; 85 /// Cache slice size. 86 23:16 cache_size; 87 /// External bus width. 88 31:24 bus_width; 89 } 90 91 /// Shader core features. 92 pub(crate) CORE_FEATURES(u32) @ 0x8 { 93 /// Shader core variant. 94 7:0 core_variant; 95 } 96 97 /// Tiler features. 98 pub(crate) TILER_FEATURES(u32) @ 0xc { 99 /// Log of the tiler's bin size. 100 5:0 bin_size; 101 /// Maximum number of active levels. 102 11:8 max_levels; 103 } 104 105 /// Memory system features. 106 pub(crate) MEM_FEATURES(u32) @ 0x10 { 107 0:0 coherent_core_group => bool; 108 1:1 coherent_super_group => bool; 109 11:8 l2_slices; 110 } 111 112 /// Memory management unit features. 113 pub(crate) MMU_FEATURES(u32) @ 0x14 { 114 /// Number of bits supported in virtual addresses. 115 7:0 va_bits; 116 /// Number of bits supported in physical addresses. 117 15:8 pa_bits; 118 } 119 120 /// Address spaces present. 121 pub(crate) AS_PRESENT(u32) @ 0x18 { 122 31:0 present; 123 } 124 125 /// CSF version information. 126 pub(crate) CSF_ID(u32) @ 0x1c { 127 /// MCU revision ID. 128 3:0 mcu_rev; 129 /// MCU minor revision number. 130 9:4 mcu_minor; 131 /// MCU major revision number. 132 15:10 mcu_major; 133 /// CSHW revision ID. 134 19:16 cshw_rev; 135 /// CSHW minor revision number. 136 25:20 cshw_minor; 137 /// CSHW major revision number. 138 31:26 cshw_major; 139 } 140 141 /// IRQ sources raw status. 142 /// Writing to this register forces bits on, but does not clear them. 143 pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { 144 /// A GPU fault has occurred, a 1-bit boolean flag. 145 0:0 gpu_fault => bool; 146 /// A GPU fault has occurred, a 1-bit boolean flag. 147 1:1 gpu_protected_fault => bool; 148 /// Reset has completed, a 1-bit boolean flag. 149 8:8 reset_completed => bool; 150 /// Set when a single power domain has powered up or down, a 1-bit boolean flag. 151 9:9 power_changed_single => bool; 152 /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. 153 10:10 power_changed_all => bool; 154 /// Set when cache cleaning has completed, a 1-bit boolean flag. 155 17:17 clean_caches_completed => bool; 156 /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. 157 18:18 doorbell_mirror => bool; 158 /// MCU requires attention, a 1-bit boolean flag. 159 19:19 mcu_status => bool; 160 } 161 162 /// IRQ sources to clear. Write only. 163 pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { 164 /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. 165 0:0 gpu_fault => bool; 166 /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 167 1:1 gpu_protected_fault => bool; 168 /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. 169 8:8 reset_completed => bool; 170 /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 171 9:9 power_changed_single => bool; 172 /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 173 10:10 power_changed_all => bool; 174 /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 175 17:17 clean_caches_completed => bool; 176 /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. 177 19:19 mcu_status => bool; 178 } 179 180 /// IRQ sources enabled. 181 pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { 182 /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. 183 0:0 gpu_fault => bool; 184 /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 185 1:1 gpu_protected_fault => bool; 186 /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. 187 8:8 reset_completed => bool; 188 /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 189 9:9 power_changed_single => bool; 190 /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 191 10:10 power_changed_all => bool; 192 /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 193 17:17 clean_caches_completed => bool; 194 /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. 195 18:18 doorbell_mirror => bool; 196 /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. 197 19:19 mcu_status => bool; 198 } 199 200 /// IRQ status for enabled sources. Read only. 201 pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { 202 /// GPU_FAULT interrupt status, a 1-bit boolean flag. 203 0:0 gpu_fault => bool; 204 /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. 205 1:1 gpu_protected_fault => bool; 206 /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. 207 8:8 reset_completed => bool; 208 /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. 209 9:9 power_changed_single => bool; 210 /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. 211 10:10 power_changed_all => bool; 212 /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. 213 17:17 clean_caches_completed => bool; 214 /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. 215 18:18 doorbell_mirror => bool; 216 /// MCU_STATUS interrupt status, a 1-bit boolean flag. 217 19:19 mcu_status => bool; 218 } 219 } 220 221 /// Helpers for GPU_COMMAND Register 222 #[derive(Copy, Clone, Debug, PartialEq)] 223 #[repr(u8)] 224 pub(crate) enum GpuCommand { 225 /// No operation. This is the default value. 226 Nop = 0, 227 /// Reset the GPU. 228 Reset = 1, 229 /// Flush caches. 230 FlushCaches = 4, 231 /// Clear GPU faults. 232 ClearFault = 7, 233 } 234 235 impl TryFrom<Bounded<u32, 8>> for GpuCommand { 236 type Error = Error; 237 238 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 239 match val.get() { 240 0 => Ok(GpuCommand::Nop), 241 1 => Ok(GpuCommand::Reset), 242 4 => Ok(GpuCommand::FlushCaches), 243 7 => Ok(GpuCommand::ClearFault), 244 _ => Err(EINVAL), 245 } 246 } 247 } 248 249 impl From<GpuCommand> for Bounded<u32, 8> { 250 fn from(cmd: GpuCommand) -> Self { 251 (cmd as u8).into() 252 } 253 } 254 255 /// Reset mode for [`GPU_COMMAND::reset()`]. 256 #[derive(Copy, Clone, Debug, PartialEq)] 257 #[repr(u8)] 258 pub(crate) enum ResetMode { 259 /// Stop all external bus interfaces, then reset the entire GPU. 260 SoftReset = 1, 261 /// Force a full GPU reset. 262 HardReset = 2, 263 } 264 265 impl TryFrom<Bounded<u32, 4>> for ResetMode { 266 type Error = Error; 267 268 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 269 match val.get() { 270 1 => Ok(ResetMode::SoftReset), 271 2 => Ok(ResetMode::HardReset), 272 _ => Err(EINVAL), 273 } 274 } 275 } 276 277 impl From<ResetMode> for Bounded<u32, 4> { 278 fn from(mode: ResetMode) -> Self { 279 Bounded::try_new(mode as u32).unwrap() 280 } 281 } 282 283 /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. 284 #[derive(Copy, Clone, Debug, PartialEq)] 285 #[repr(u8)] 286 pub(crate) enum FlushMode { 287 /// No flush. 288 None = 0, 289 /// Clean the caches. 290 Clean = 1, 291 /// Invalidate the caches. 292 Invalidate = 2, 293 /// Clean and invalidate the caches. 294 CleanInvalidate = 3, 295 } 296 297 impl TryFrom<Bounded<u32, 4>> for FlushMode { 298 type Error = Error; 299 300 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 301 match val.get() { 302 0 => Ok(FlushMode::None), 303 1 => Ok(FlushMode::Clean), 304 2 => Ok(FlushMode::Invalidate), 305 3 => Ok(FlushMode::CleanInvalidate), 306 _ => Err(EINVAL), 307 } 308 } 309 } 310 311 impl From<FlushMode> for Bounded<u32, 4> { 312 fn from(mode: FlushMode) -> Self { 313 Bounded::try_new(mode as u32).unwrap() 314 } 315 } 316 317 register! { 318 /// GPU command register. 319 /// 320 /// Use the constructor methods to create commands: 321 /// - [`GPU_COMMAND::nop()`] 322 /// - [`GPU_COMMAND::reset()`] 323 /// - [`GPU_COMMAND::flush_caches()`] 324 /// - [`GPU_COMMAND::clear_fault()`] 325 pub(crate) GPU_COMMAND (u32) @ 0x30 { 326 7:0 command ?=> GpuCommand; 327 } 328 /// Internal alias for GPU_COMMAND in reset mode. 329 /// Use [`GPU_COMMAND::reset()`] instead. 330 GPU_COMMAND_RESET (u32) => GPU_COMMAND { 331 7:0 command ?=> GpuCommand; 332 11:8 reset_mode ?=> ResetMode; 333 } 334 335 /// Internal alias for GPU_COMMAND in cache flush mode. 336 /// Use [`GPU_COMMAND::flush_caches()`] instead. 337 GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { 338 7:0 command ?=> GpuCommand; 339 /// L2 cache flush mode. 340 11:8 l2_flush ?=> FlushMode; 341 /// Shader core load/store cache flush mode. 342 15:12 lsc_flush ?=> FlushMode; 343 /// Shader core other caches flush mode. 344 19:16 other_flush ?=> FlushMode; 345 } 346 } 347 348 impl GPU_COMMAND { 349 /// Create a NOP command. 350 pub(crate) fn nop() -> Self { 351 Self::zeroed() 352 } 353 354 /// Create a reset command with the specified reset mode. 355 pub(crate) fn reset(mode: ResetMode) -> Self { 356 Self::from_raw( 357 GPU_COMMAND_RESET::zeroed() 358 .with_command(GpuCommand::Reset) 359 .with_reset_mode(mode) 360 .into_raw(), 361 ) 362 } 363 364 /// Create a cache flush command with the specified flush modes. 365 pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { 366 Self::from_raw( 367 GPU_COMMAND_FLUSH::zeroed() 368 .with_command(GpuCommand::FlushCaches) 369 .with_l2_flush(l2) 370 .with_lsc_flush(lsc) 371 .with_other_flush(other) 372 .into_raw(), 373 ) 374 } 375 376 /// Create a clear fault command. 377 pub(crate) fn clear_fault() -> Self { 378 Self::zeroed().with_command(GpuCommand::ClearFault) 379 } 380 } 381 382 register! { 383 /// GPU status register. Read only. 384 pub(crate) GPU_STATUS(u32) @ 0x34 { 385 /// GPU active, a 1-bit boolean flag. 386 0:0 gpu_active => bool; 387 /// Power manager active, a 1-bit boolean flag 388 1:1 pwr_active => bool; 389 /// Page fault active, a 1-bit boolean flag. 390 4:4 page_fault => bool; 391 /// Protected mode active, a 1-bit boolean flag. 392 7:7 protected_mode_active => bool; 393 /// Debug mode active, a 1-bit boolean flag. 394 8:8 gpu_dbg_enabled => bool; 395 } 396 } 397 398 #[derive(Copy, Clone, Debug, PartialEq)] 399 #[repr(u8)] 400 pub(crate) enum ExceptionType { 401 /// Exception type: No error. 402 Ok = 0x00, 403 /// Exception type: GPU external bus error. 404 GpuBusFault = 0x80, 405 /// Exception type: GPU shareability error. 406 GpuShareabilityFault = 0x88, 407 /// Exception type: System shareability error. 408 SystemShareabilityFault = 0x89, 409 /// Exception type: GPU cacheability error. 410 GpuCacheabilityFault = 0x8A, 411 } 412 413 impl TryFrom<Bounded<u32, 8>> for ExceptionType { 414 type Error = Error; 415 416 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 417 match val.get() { 418 0x00 => Ok(ExceptionType::Ok), 419 0x80 => Ok(ExceptionType::GpuBusFault), 420 0x88 => Ok(ExceptionType::GpuShareabilityFault), 421 0x89 => Ok(ExceptionType::SystemShareabilityFault), 422 0x8A => Ok(ExceptionType::GpuCacheabilityFault), 423 _ => Err(EINVAL), 424 } 425 } 426 } 427 428 impl From<ExceptionType> for Bounded<u32, 8> { 429 fn from(exc: ExceptionType) -> Self { 430 (exc as u8).into() 431 } 432 } 433 434 #[derive(Copy, Clone, Debug, PartialEq)] 435 #[repr(u8)] 436 pub(crate) enum AccessType { 437 /// Access type: An atomic (read/write) transaction. 438 Atomic = 0, 439 /// Access type: An execute transaction. 440 Execute = 1, 441 /// Access type: A read transaction. 442 Read = 2, 443 /// Access type: A write transaction. 444 Write = 3, 445 } 446 447 impl From<Bounded<u32, 2>> for AccessType { 448 fn from(val: Bounded<u32, 2>) -> Self { 449 match val.get() { 450 0 => AccessType::Atomic, 451 1 => AccessType::Execute, 452 2 => AccessType::Read, 453 3 => AccessType::Write, 454 _ => unreachable!(), 455 } 456 } 457 } 458 459 impl From<AccessType> for Bounded<u32, 2> { 460 fn from(access: AccessType) -> Self { 461 Bounded::try_new(access as u32).unwrap() 462 } 463 } 464 465 register! { 466 /// GPU fault status register. Read only. 467 pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { 468 /// Exception type. 469 7:0 exception_type ?=> ExceptionType; 470 /// Access type. 471 9:8 access_type => AccessType; 472 /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. 473 10:10 address_valid => bool; 474 /// The JASID field is valid, a 1-bit boolean flag. 475 11:11 jasid_valid => bool; 476 /// JASID of the fault, if known. 477 15:12 jasid; 478 /// ID of the source that triggered the fault. 479 31:16 source_id; 480 } 481 482 /// GPU fault address. Read only. 483 /// Once a fault is reported, it must be manually cleared by issuing a 484 /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU 485 /// faults will be reported until the previous fault has been cleared. 486 pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { 487 31:0 pointer; 488 } 489 490 pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { 491 31:0 pointer; 492 } 493 494 /// Level 2 cache configuration. 495 pub(crate) L2_CONFIG(u32) @ 0x48 { 496 /// Requested cache size. 497 23:16 cache_size; 498 /// Requested hash function index. 499 31:24 hash_function; 500 } 501 502 /// Global time stamp offset. 503 pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { 504 31:0 offset; 505 } 506 507 pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { 508 31:0 offset; 509 } 510 511 /// GPU cycle counter. Read only. 512 pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { 513 31:0 count; 514 } 515 516 pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { 517 31:0 count; 518 } 519 520 /// Global time stamp. Read only. 521 pub(crate) TIMESTAMP_LO(u32) @ 0x98 { 522 31:0 timestamp; 523 } 524 525 pub(crate) TIMESTAMP_HI(u32) @ 0x9c { 526 31:0 timestamp; 527 } 528 529 /// Maximum number of threads per core. Read only constant. 530 pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { 531 31:0 threads; 532 } 533 534 /// Maximum number of threads per workgroup. Read only constant. 535 pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { 536 31:0 threads; 537 } 538 539 /// Maximum number of threads per barrier. Read only constant. 540 pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { 541 31:0 threads; 542 } 543 544 /// Thread features. Read only constant. 545 pub(crate) THREAD_FEATURES(u32) @ 0xac { 546 /// Total number of registers per core. 547 21:0 max_registers; 548 /// Implementation technology type. 549 23:22 implementation_technology; 550 /// Maximum number of compute tasks waiting. 551 31:24 max_task_queue; 552 } 553 554 /// Support flags for compressed texture formats. Read only constant. 555 /// 556 /// A bitmap where each bit indicates support for a specific compressed texture format. 557 /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). 558 pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { 559 31:0 supported_formats; 560 } 561 562 /// Shader core present bitmap. Read only constant. 563 pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { 564 31:0 value; 565 } 566 567 pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { 568 31:0 value; 569 } 570 571 /// Tiler present bitmap. Read only constant. 572 pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { 573 31:0 present; 574 } 575 576 pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { 577 31:0 present; 578 } 579 580 /// L2 cache present bitmap. Read only constant. 581 pub(crate) L2_PRESENT_LO(u32) @ 0x120 { 582 31:0 present; 583 } 584 585 pub(crate) L2_PRESENT_HI(u32) @ 0x124 { 586 31:0 present; 587 } 588 589 /// Shader core ready bitmap. Read only. 590 pub(crate) SHADER_READY_LO(u32) @ 0x140 { 591 31:0 ready; 592 } 593 594 pub(crate) SHADER_READY_HI(u32) @ 0x144 { 595 31:0 ready; 596 } 597 598 /// Tiler ready bitmap. Read only. 599 pub(crate) TILER_READY_LO(u32) @ 0x150 { 600 31:0 ready; 601 } 602 603 pub(crate) TILER_READY_HI(u32) @ 0x154 { 604 31:0 ready; 605 } 606 607 /// L2 ready bitmap. Read only. 608 pub(crate) L2_READY_LO(u32) @ 0x160 { 609 31:0 ready; 610 } 611 612 pub(crate) L2_READY_HI(u32) @ 0x164 { 613 31:0 ready; 614 } 615 616 /// Shader core power up bitmap. 617 pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { 618 31:0 request; 619 } 620 621 pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { 622 31:0 request; 623 } 624 625 /// Tiler power up bitmap. 626 pub(crate) TILER_PWRON_LO(u32) @ 0x190 { 627 31:0 request; 628 } 629 630 pub(crate) TILER_PWRON_HI(u32) @ 0x194 { 631 31:0 request; 632 } 633 634 /// L2 power up bitmap. 635 pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { 636 31:0 request; 637 } 638 639 pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { 640 31:0 request; 641 } 642 643 /// Shader core power down bitmap. 644 pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { 645 31:0 request; 646 } 647 648 pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { 649 31:0 request; 650 } 651 652 /// Tiler power down bitmap. 653 pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { 654 31:0 request; 655 } 656 657 pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { 658 31:0 request; 659 } 660 661 /// L2 power down bitmap. 662 pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { 663 31:0 request; 664 } 665 666 pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { 667 31:0 request; 668 } 669 670 /// Shader core power transition bitmap. Read-only. 671 pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { 672 31:0 changing; 673 } 674 675 pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { 676 31:0 changing; 677 } 678 679 /// Tiler power transition bitmap. Read-only. 680 pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { 681 31:0 changing; 682 } 683 684 pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { 685 31:0 changing; 686 } 687 688 /// L2 power transition bitmap. Read-only. 689 pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { 690 31:0 changing; 691 } 692 693 pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { 694 31:0 changing; 695 } 696 697 /// Shader core active bitmap. Read-only. 698 pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { 699 31:0 active; 700 } 701 702 pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { 703 31:0 active; 704 } 705 706 /// Tiler active bitmap. Read-only. 707 pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { 708 31:0 active; 709 } 710 711 pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { 712 31:0 active; 713 } 714 715 /// L2 active bitmap. Read-only. 716 pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { 717 31:0 active; 718 } 719 720 pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { 721 31:0 active; 722 } 723 724 /// Revision ID. Read only constant. 725 pub(crate) REVIDR(u32) @ 0x280 { 726 31:0 revision; 727 } 728 729 /// Coherency features present. Read only constant. 730 /// Supported protocols on the interconnect between the GPU and the 731 /// system into which it is integrated. 732 pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { 733 /// ACE-Lite protocol supported, a 1-bit boolean flag. 734 0:0 ace_lite => bool; 735 /// ACE protocol supported, a 1-bit boolean flag. 736 1:1 ace => bool; 737 } 738 } 739 740 #[derive(Copy, Clone, Debug, PartialEq)] 741 #[repr(u8)] 742 pub(crate) enum CoherencyMode { 743 /// ACE-Lite coherency protocol. 744 AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, 745 /// ACE coherency protocol. 746 Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, 747 /// No coherency protocol. 748 None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, 749 } 750 751 impl TryFrom<Bounded<u32, 32>> for CoherencyMode { 752 type Error = Error; 753 754 fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> { 755 match val.get() { 756 0 => Ok(CoherencyMode::AceLite), 757 1 => Ok(CoherencyMode::Ace), 758 31 => Ok(CoherencyMode::None), 759 _ => Err(EINVAL), 760 } 761 } 762 } 763 764 impl From<CoherencyMode> for Bounded<u32, 32> { 765 fn from(mode: CoherencyMode) -> Self { 766 (mode as u8).into() 767 } 768 } 769 770 register! { 771 /// Coherency enable. An index of which coherency protocols should be used. 772 /// This register only selects the protocol for coherency messages on the 773 /// interconnect. This is not to enable or disable coherency controlled by MMU. 774 pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { 775 31:0 l2_cache_protocol_select ?=> CoherencyMode; 776 } 777 } 778 779 /// Helpers for MCU_CONTROL register 780 #[derive(Copy, Clone, Debug, PartialEq)] 781 #[repr(u8)] 782 pub(crate) enum McuControlMode { 783 /// Disable the MCU. 784 Disable = 0, 785 /// Enable the MCU. 786 Enable = 1, 787 /// Enable the MCU to execute and automatically reboot after a fast reset. 788 Auto = 2, 789 } 790 791 impl TryFrom<Bounded<u32, 2>> for McuControlMode { 792 type Error = Error; 793 794 fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> { 795 match val.get() { 796 0 => Ok(McuControlMode::Disable), 797 1 => Ok(McuControlMode::Enable), 798 2 => Ok(McuControlMode::Auto), 799 _ => Err(EINVAL), 800 } 801 } 802 } 803 804 impl From<McuControlMode> for Bounded<u32, 2> { 805 fn from(mode: McuControlMode) -> Self { 806 Bounded::try_new(mode as u32).unwrap() 807 } 808 } 809 810 register! { 811 /// MCU control. 812 pub(crate) MCU_CONTROL(u32) @ 0x700 { 813 /// Request MCU state change. 814 1:0 req ?=> McuControlMode; 815 } 816 } 817 818 /// Helpers for MCU_STATUS register 819 #[derive(Copy, Clone, Debug, PartialEq)] 820 #[repr(u8)] 821 pub(crate) enum McuStatus { 822 /// MCU is disabled. 823 Disabled = 0, 824 /// MCU is enabled. 825 Enabled = 1, 826 /// The MCU has halted by itself in an orderly manner to enable the core group to be 827 /// powered down. 828 Halt = 2, 829 /// The MCU has encountered an error that prevents it from continuing. 830 Fatal = 3, 831 } 832 833 impl From<Bounded<u32, 2>> for McuStatus { 834 fn from(val: Bounded<u32, 2>) -> Self { 835 match val.get() { 836 0 => McuStatus::Disabled, 837 1 => McuStatus::Enabled, 838 2 => McuStatus::Halt, 839 3 => McuStatus::Fatal, 840 _ => unreachable!(), 841 } 842 } 843 } 844 845 impl From<McuStatus> for Bounded<u32, 2> { 846 fn from(status: McuStatus) -> Self { 847 Bounded::try_new(status as u32).unwrap() 848 } 849 } 850 851 register! { 852 /// MCU status. Read only. 853 pub(crate) MCU_STATUS(u32) @ 0x704 { 854 /// Read current state of MCU. 855 1:0 value => McuStatus; 856 } 857 } 858 } 859 860 /// These registers correspond to the JOB_CONTROL register page. 861 /// They are involved in communication between the firmware running on the MCU and the host. 862 pub(crate) mod job_control { 863 use kernel::register; 864 865 register! { 866 /// Raw status of job interrupts. 867 /// 868 /// Write to this register to trigger these interrupts. 869 /// Writing a 1 to a bit forces that bit on. 870 pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { 871 /// CSG request. These bits indicate that CSGn requires attention from the host. 872 30:0 csg; 873 /// GLB request. Indicates that the GLB interface requires attention from the host. 874 31:31 glb => bool; 875 } 876 877 /// Clear job interrupts. Write only. 878 /// 879 /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. 880 pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { 881 /// Clear CSG request interrupts. 882 30:0 csg; 883 /// Clear GLB request interrupt. 884 31:31 glb => bool; 885 } 886 887 /// Mask for job interrupts. 888 /// 889 /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. 890 pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { 891 /// Enable CSG request interrupts. 892 30:0 csg; 893 /// Enable GLB request interrupt. 894 31:31 glb => bool; 895 } 896 897 /// Active job interrupts. Read only. 898 /// 899 /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and 900 /// [`JOB_IRQ_MASK`]. 901 pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { 902 /// CSG request interrupt status. 903 30:0 csg; 904 /// GLB request interrupt status. 905 31:31 glb => bool; 906 } 907 } 908 } 909 910 /// These registers correspond to the MMU_CONTROL register page. 911 /// They are involved in MMU configuration and control. 912 pub(crate) mod mmu_control { 913 use kernel::register; 914 915 register! { 916 /// IRQ sources raw status. 917 /// 918 /// This register contains the raw unmasked interrupt sources for MMU status and exception 919 /// handling. 920 /// 921 /// Writing to this register forces bits on. 922 /// Use [`IRQ_CLEAR`] to clear interrupts. 923 pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { 924 /// Page fault for address spaces. 925 15:0 page_fault; 926 /// Command completed in address spaces. 927 31:16 command_completed; 928 } 929 930 /// IRQ sources to clear. 931 /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. 932 pub(crate) IRQ_CLEAR(u32) @ 0x2004 { 933 /// Clear the PAGE_FAULT interrupt. 934 15:0 page_fault; 935 /// Clear the COMMAND_COMPLETED interrupt. 936 31:16 command_completed; 937 } 938 939 /// IRQ sources enabled. 940 /// 941 /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. 942 pub(crate) IRQ_MASK(u32) @ 0x2008 { 943 /// Enable the PAGE_FAULT interrupt. 944 15:0 page_fault; 945 /// Enable the COMMAND_COMPLETED interrupt. 946 31:16 command_completed; 947 } 948 949 /// IRQ status for enabled sources. Read only. 950 /// 951 /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. 952 pub(crate) IRQ_STATUS(u32) @ 0x200c { 953 /// PAGE_FAULT interrupt status. 954 15:0 page_fault; 955 /// COMMAND_COMPLETED interrupt status. 956 31:16 command_completed; 957 } 958 } 959 960 /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. 961 /// 962 /// This array contains 16 instances of the MMU_AS_CONTROL register page. 963 pub(crate) mod mmu_as_control { 964 use kernel::{ 965 num::Bounded, 966 prelude::*, 967 register, // 968 }; 969 970 use pin_init::Zeroable; 971 972 /// Maximum number of hardware address space slots. 973 /// The actual number of slots available is usually lower. 974 pub(crate) const MAX_AS: usize = 16; 975 976 /// Address space register stride. The elements in the array are spaced 64B apart. 977 const STRIDE: usize = 0x40; 978 979 register! { 980 /// Translation table base address. A 64-bit pointer. 981 /// 982 /// This field contains the address of the top level of a translation table structure. 983 /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. 984 pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { 985 /// Base address of the translation table. 986 63:0 base; 987 } 988 989 // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two 990 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 991 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 992 pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { 993 31:0 value; 994 } 995 996 pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { 997 31:0 value; 998 } 999 } 1000 1001 /// Helpers for MEMATTR Register. 1002 1003 #[derive(Copy, Clone, Debug, PartialEq)] 1004 #[repr(u8)] 1005 pub(crate) enum AllocPolicySelect { 1006 /// Ignore ALLOC_R/ALLOC_W fields. 1007 Impl = 2, 1008 /// Use ALLOC_R/ALLOC_W fields for allocation policy. 1009 Alloc = 3, 1010 } 1011 1012 impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect { 1013 type Error = Error; 1014 1015 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1016 match val.get() { 1017 2 => Ok(Self::Impl), 1018 3 => Ok(Self::Alloc), 1019 _ => Err(EINVAL), 1020 } 1021 } 1022 } 1023 1024 impl From<AllocPolicySelect> for Bounded<u8, 2> { 1025 fn from(val: AllocPolicySelect) -> Self { 1026 Bounded::try_new(val as u8).unwrap() 1027 } 1028 } 1029 1030 /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. 1031 /// 1032 /// The hardware spec defines different interpretations of these values depending on 1033 /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation 1034 /// tables (all input addresses map to the same output address); it is deprecated and not 1035 /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. 1036 #[derive(Copy, Clone, Debug, PartialEq)] 1037 #[repr(u8)] 1038 pub(crate) enum Coherency { 1039 /// Midgard inner domain coherency. 1040 /// 1041 /// Most flexible mode - can map non-coherent, internally coherent, and system/IO 1042 /// coherent memory. Used for non-cacheable memory in MAIR conversion. 1043 MidgardInnerDomain = 0, 1044 /// CPU inner domain coherency. 1045 /// 1046 /// Can map non-coherent and system/IO coherent memory. Used for write-back 1047 /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. 1048 CpuInnerDomain = 1, 1049 /// CPU inner domain with shader coherency. 1050 /// 1051 /// Can map internally coherent and system/IO coherent memory. Used for 1052 /// GPU-internal shared buffers requiring shader coherency. 1053 CpuInnerDomainShaderCoh = 2, 1054 } 1055 1056 impl TryFrom<Bounded<u8, 2>> for Coherency { 1057 type Error = Error; 1058 1059 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1060 match val.get() { 1061 0 => Ok(Self::MidgardInnerDomain), 1062 1 => Ok(Self::CpuInnerDomain), 1063 2 => Ok(Self::CpuInnerDomainShaderCoh), 1064 _ => Err(EINVAL), 1065 } 1066 } 1067 } 1068 1069 impl From<Coherency> for Bounded<u8, 2> { 1070 fn from(val: Coherency) -> Self { 1071 Bounded::try_new(val as u8).unwrap() 1072 } 1073 } 1074 1075 #[derive(Copy, Clone, Debug, PartialEq)] 1076 #[repr(u8)] 1077 pub(crate) enum MemoryType { 1078 /// Normal memory (shared). 1079 Shared = 0, 1080 /// Normal memory, inner/outer non-cacheable. 1081 NonCacheable = 1, 1082 /// Normal memory, inner/outer write-back cacheable. 1083 WriteBack = 2, 1084 /// Triggers MEMORY_ATTRIBUTE_FAULT. 1085 Fault = 3, 1086 } 1087 1088 impl From<Bounded<u8, 2>> for MemoryType { 1089 fn from(val: Bounded<u8, 2>) -> Self { 1090 match val.get() { 1091 0 => Self::Shared, 1092 1 => Self::NonCacheable, 1093 2 => Self::WriteBack, 1094 3 => Self::Fault, 1095 _ => unreachable!(), 1096 } 1097 } 1098 } 1099 1100 impl From<MemoryType> for Bounded<u8, 2> { 1101 fn from(val: MemoryType) -> Self { 1102 Bounded::try_new(val as u8).unwrap() 1103 } 1104 } 1105 1106 register! { 1107 /// Stage 1 memory attributes (8-bit bitfield). 1108 /// 1109 /// This is not an actual register, but a bitfield definition used by the MEMATTR 1110 /// register. Each of the 8 bytes in MEMATTR follows this layout. 1111 MMU_MEMATTR_STAGE1(u8) @ 0x0 { 1112 /// Inner cache write allocation policy. 1113 0:0 alloc_w => bool; 1114 /// Inner cache read allocation policy. 1115 1:1 alloc_r => bool; 1116 /// Inner allocation policy select. 1117 3:2 alloc_sel ?=> AllocPolicySelect; 1118 /// Coherency policy. 1119 5:4 coherency ?=> Coherency; 1120 /// Memory type. 1121 7:6 memory_type => MemoryType; 1122 } 1123 } 1124 1125 impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 { 1126 type Error = Error; 1127 1128 fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> { 1129 Ok(Self::from_raw(val.get() as u8)) 1130 } 1131 } 1132 1133 impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> { 1134 fn from(val: MMU_MEMATTR_STAGE1) -> Self { 1135 Bounded::try_new(u64::from(val.into_raw())).unwrap() 1136 } 1137 } 1138 1139 register! { 1140 /// Memory attributes. 1141 /// 1142 /// Each address space can configure up to 8 different memory attribute profiles. 1143 /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. 1144 pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { 1145 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; 1146 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; 1147 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; 1148 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; 1149 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; 1150 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; 1151 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; 1152 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; 1153 } 1154 1155 // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two 1156 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1157 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1158 pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { 1159 31:0 value; 1160 } 1161 1162 pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { 1163 31:0 value; 1164 } 1165 } 1166 1167 impl MEMATTR { 1168 /// Outer cache-policy nibble indicating device memory. 1169 const ARM_MAIR_DEVICE_MEMORY: u8 = 0x0; 1170 1171 /// In the ARM Architecture Reference Manual, the MAIR encoding for Normal memory 1172 /// uses the format `0bxxRW` where: 1173 /// - `W` (bit 0) = Write-Allocate policy 1174 /// - `R` (bit 1) = Read-Allocate policy 1175 /// E.g., `0b0011` would allow both read and write allocation on a cache miss. 1176 /// 1177 /// ARM MAIR Write-Allocate bit (bit 0 of a cache policy nibble). 1178 const ARM_MAIR_WRITE_ALLOCATE: u8 = 0x1; 1179 /// ARM MAIR Read-Allocate bit (bit 1 of a cache policy nibble). 1180 const ARM_MAIR_READ_ALLOCATE: u8 = 0x2; 1181 1182 /// Write-back policy bit. For cacheable encodings, it is necessary but not 1183 /// sufficient to set bit 2 of the cache policy nibble. Bit 2 does not 1184 /// definitively determine write back because bit 2 is also set in `0b0100` 1185 /// which encodes Normal non-cacheable memory. 1186 const ARM_MAIR_WRITE_BACK_BIT: u8 = 0x4; 1187 1188 /// Complete cache-policy nibble encoding for Normal Non-cacheable memory. 1189 const ARM_MAIR_NON_CACHEABLE: u8 = 0x4; 1190 1191 /// Mask for the inner cache policy nibble in MAIR attribute bytes. 1192 const ARM_MAIR_INNER_MASK: u8 = 0x0f; 1193 1194 /// Check if a MAIR attribute byte represents device memory. 1195 /// 1196 /// Device memory (memory-mapped I/O, registers) cannot be cached because 1197 /// reading and writing to this memory may have side effects. 1198 fn is_device_memory(mair_attr: u8) -> bool { 1199 // In AArch64 MAIR, outer nibble only is 0 for device memory. 1200 (mair_attr >> 4) == Self::ARM_MAIR_DEVICE_MEMORY 1201 } 1202 1203 /// Check if normal memory is fully write-back cacheable. 1204 /// 1205 /// ARM MAIR has two cache policy levels (outer [7:4] and inner [3:0]). 1206 /// For memory to be truly write-back, BOTH levels must have the write-back bit set. 1207 /// If only one level is write-back, treat it as non-cacheable for GPU purposes. 1208 fn is_writeback_cacheable(mair_attr: u8) -> bool { 1209 let outer = mair_attr >> 4; 1210 let inner = mair_attr & Self::ARM_MAIR_INNER_MASK; 1211 1212 outer != Self::ARM_MAIR_NON_CACHEABLE 1213 && inner != Self::ARM_MAIR_NON_CACHEABLE 1214 && (outer & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 1215 && (inner & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 1216 } 1217 1218 // Helper to encode a MEMATTR attribute from its individual fields. 1219 fn encode_attribute( 1220 alloc_w: bool, 1221 alloc_r: bool, 1222 alloc_sel: AllocPolicySelect, 1223 coherency: Coherency, 1224 memory_type: MemoryType, 1225 ) -> MMU_MEMATTR_STAGE1 { 1226 MMU_MEMATTR_STAGE1::zeroed() 1227 .with_alloc_w(alloc_w) 1228 .with_alloc_r(alloc_r) 1229 .with_alloc_sel(alloc_sel) 1230 .with_coherency(coherency) 1231 .with_memory_type(memory_type) 1232 } 1233 1234 /// Convert one MAIR attribute byte into a MEMATTR attribute. 1235 // TODO: Add a `coherent` parameter like panthor's mair_to_memattr(). 1236 // For now, assume a non-coherent system and always encode write-back 1237 // memory with MidgardInnerDomain coherency. 1238 fn attribute_from_mair(mair_attr: u8) -> MMU_MEMATTR_STAGE1 { 1239 // Device memory or non-write-back normal memory 1240 if Self::is_device_memory(mair_attr) || !Self::is_writeback_cacheable(mair_attr) { 1241 return Self::encode_attribute( 1242 false, 1243 false, 1244 AllocPolicySelect::Alloc, 1245 Coherency::MidgardInnerDomain, 1246 MemoryType::NonCacheable, 1247 ); 1248 } 1249 1250 // Write-back cacheable normal memory 1251 let inner: u8 = mair_attr & Self::ARM_MAIR_INNER_MASK; 1252 Self::encode_attribute( 1253 (inner & Self::ARM_MAIR_WRITE_ALLOCATE) != 0, 1254 (inner & Self::ARM_MAIR_READ_ALLOCATE) != 0, 1255 AllocPolicySelect::Alloc, 1256 Coherency::MidgardInnerDomain, 1257 MemoryType::WriteBack, 1258 ) 1259 } 1260 1261 /// Write one converted MAIR attribute into a corresponding MEMATTR slot. 1262 fn with_encoded_attribute(self, index: usize, attr: MMU_MEMATTR_STAGE1) -> Self { 1263 debug_assert!(index < 8); 1264 1265 let shift = index * 8; 1266 let mask = !(0xffu64 << shift); 1267 let raw = (self.into_raw() & mask) | ((u64::from(attr.into_raw())) << shift); 1268 1269 Self::from_raw(raw) 1270 } 1271 1272 /// Convert an AArch64 MAIR value into the GPU MEMATTR register encoding. 1273 /// 1274 /// Both MAIR and MEMATTR are 64-bit values with eight 8-bit memory 1275 /// attribute entries, but the bits do not map directly. The GPU MEMATTR encoding 1276 /// is less detailed than the MAIR encoding, so MAIR is converted to MEMATTR 1277 /// conservatively as follows: 1278 /// 1279 /// 1. Device memory, or Normal Memory that is not write-back cacheable, is encoded 1280 /// as GPU `NonCacheable` 1281 /// 1282 /// 2. Normal memory that is write-back cacheable is encoded as GPU `WriteBack`, 1283 /// and the inner allocation hints are preserved. 1284 pub(crate) fn from_mair(mair: u64) -> Self { 1285 mair.to_le_bytes() 1286 .into_iter() 1287 .enumerate() 1288 .fold(Self::zeroed(), |acc, (i, attr)| { 1289 acc.with_encoded_attribute(i, Self::attribute_from_mair(attr)) 1290 }) 1291 } 1292 } 1293 1294 register! { 1295 /// Lock region address for each address space. 1296 pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { 1297 /// Lock region size. 1298 5:0 size; 1299 /// Lock region base address. 1300 63:12 base; 1301 } 1302 1303 // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two 1304 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1305 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1306 pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { 1307 31:0 value; 1308 } 1309 1310 pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { 1311 31:0 value; 1312 } 1313 } 1314 1315 /// Helpers for MMU COMMAND register. 1316 #[derive(Copy, Clone, Debug, PartialEq)] 1317 #[repr(u8)] 1318 pub(crate) enum MmuCommand { 1319 /// No operation, nothing happens. 1320 Nop = 0, 1321 /// Propagate settings to the MMU. 1322 Update = 1, 1323 /// Lock an address region. 1324 Lock = 2, 1325 /// Unlock an address region. 1326 Unlock = 3, 1327 /// Clean and invalidate the L2 cache, then unlock. 1328 FlushPt = 4, 1329 /// Clean and invalidate all caches, then unlock. 1330 FlushMem = 5, 1331 } 1332 1333 impl TryFrom<Bounded<u32, 8>> for MmuCommand { 1334 type Error = Error; 1335 1336 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1337 match val.get() { 1338 0 => Ok(MmuCommand::Nop), 1339 1 => Ok(MmuCommand::Update), 1340 2 => Ok(MmuCommand::Lock), 1341 3 => Ok(MmuCommand::Unlock), 1342 4 => Ok(MmuCommand::FlushPt), 1343 5 => Ok(MmuCommand::FlushMem), 1344 _ => Err(EINVAL), 1345 } 1346 } 1347 } 1348 1349 impl From<MmuCommand> for Bounded<u32, 8> { 1350 fn from(cmd: MmuCommand) -> Self { 1351 (cmd as u8).into() 1352 } 1353 } 1354 1355 register! { 1356 /// MMU command register for each address space. Write only. 1357 pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 1358 7:0 command ?=> MmuCommand; 1359 } 1360 } 1361 1362 /// MMU exception types for FAULTSTATUS register. 1363 #[derive(Copy, Clone, Debug, PartialEq)] 1364 #[repr(u8)] 1365 pub(crate) enum MmuExceptionType { 1366 /// No error. 1367 Ok = 0x00, 1368 /// Invalid translation table entry, level 0. 1369 TranslationFault0 = 0xC0, 1370 /// Invalid translation table entry, level 1. 1371 TranslationFault1 = 0xC1, 1372 /// Invalid translation table entry, level 2. 1373 TranslationFault2 = 0xC2, 1374 /// Invalid translation table entry, level 3. 1375 TranslationFault3 = 0xC3, 1376 /// Invalid block descriptor. 1377 TranslationFault4 = 0xC4, 1378 /// Page permission error, level 0. 1379 PermissionFault0 = 0xC8, 1380 /// Page permission error, level 1. 1381 PermissionFault1 = 0xC9, 1382 /// Page permission error, level 2. 1383 PermissionFault2 = 0xCA, 1384 /// Page permission error, level 3. 1385 PermissionFault3 = 0xCB, 1386 /// Access flag not set, level 1. 1387 AccessFlag1 = 0xD9, 1388 /// Access flag not set, level 2. 1389 AccessFlag2 = 0xDA, 1390 /// Access flag not set, level 3. 1391 AccessFlag3 = 0xDB, 1392 /// Virtual address out of range. 1393 AddressSizeFaultIn = 0xE0, 1394 /// Physical address out of range, level 0. 1395 AddressSizeFaultOut0 = 0xE4, 1396 /// Physical address out of range, level 1. 1397 AddressSizeFaultOut1 = 0xE5, 1398 /// Physical address out of range, level 2. 1399 AddressSizeFaultOut2 = 0xE6, 1400 /// Physical address out of range, level 3. 1401 AddressSizeFaultOut3 = 0xE7, 1402 /// Page attribute error, level 0. 1403 MemoryAttributeFault0 = 0xE8, 1404 /// Page attribute error, level 1. 1405 MemoryAttributeFault1 = 0xE9, 1406 /// Page attribute error, level 2. 1407 MemoryAttributeFault2 = 0xEA, 1408 /// Page attribute error, level 3. 1409 MemoryAttributeFault3 = 0xEB, 1410 } 1411 1412 impl TryFrom<Bounded<u32, 8>> for MmuExceptionType { 1413 type Error = Error; 1414 1415 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1416 match val.get() { 1417 0x00 => Ok(MmuExceptionType::Ok), 1418 0xC0 => Ok(MmuExceptionType::TranslationFault0), 1419 0xC1 => Ok(MmuExceptionType::TranslationFault1), 1420 0xC2 => Ok(MmuExceptionType::TranslationFault2), 1421 0xC3 => Ok(MmuExceptionType::TranslationFault3), 1422 0xC4 => Ok(MmuExceptionType::TranslationFault4), 1423 0xC8 => Ok(MmuExceptionType::PermissionFault0), 1424 0xC9 => Ok(MmuExceptionType::PermissionFault1), 1425 0xCA => Ok(MmuExceptionType::PermissionFault2), 1426 0xCB => Ok(MmuExceptionType::PermissionFault3), 1427 0xD9 => Ok(MmuExceptionType::AccessFlag1), 1428 0xDA => Ok(MmuExceptionType::AccessFlag2), 1429 0xDB => Ok(MmuExceptionType::AccessFlag3), 1430 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), 1431 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), 1432 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), 1433 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), 1434 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), 1435 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), 1436 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), 1437 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), 1438 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), 1439 _ => Err(EINVAL), 1440 } 1441 } 1442 } 1443 1444 impl From<MmuExceptionType> for Bounded<u32, 8> { 1445 fn from(exc: MmuExceptionType) -> Self { 1446 (exc as u8).into() 1447 } 1448 } 1449 1450 /// Access type for MMU faults. 1451 #[derive(Copy, Clone, Debug, PartialEq)] 1452 #[repr(u8)] 1453 pub(crate) enum MmuAccessType { 1454 /// An atomic (read/write) transaction. 1455 Atomic = 0, 1456 /// An execute transaction. 1457 Execute = 1, 1458 /// A read transaction. 1459 Read = 2, 1460 /// A write transaction. 1461 Write = 3, 1462 } 1463 1464 impl From<Bounded<u32, 2>> for MmuAccessType { 1465 fn from(val: Bounded<u32, 2>) -> Self { 1466 match val.get() { 1467 0 => MmuAccessType::Atomic, 1468 1 => MmuAccessType::Execute, 1469 2 => MmuAccessType::Read, 1470 3 => MmuAccessType::Write, 1471 _ => unreachable!(), 1472 } 1473 } 1474 } 1475 1476 impl From<MmuAccessType> for Bounded<u32, 2> { 1477 fn from(access: MmuAccessType) -> Self { 1478 Bounded::try_new(access as u32).unwrap() 1479 } 1480 } 1481 1482 register! { 1483 /// Fault status register for each address space. Read only. 1484 pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { 1485 /// Exception type. 1486 7:0 exception_type ?=> MmuExceptionType; 1487 /// Access type. 1488 9:8 access_type => MmuAccessType; 1489 /// ID of the source that triggered the fault. 1490 31:16 source_id; 1491 } 1492 1493 /// Fault address for each address space. Read only. 1494 pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { 1495 31:0 pointer; 1496 } 1497 1498 pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { 1499 31:0 pointer; 1500 } 1501 1502 /// MMU status register for each address space. Read only. 1503 pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { 1504 /// External address space command is active, a 1-bit boolean flag. 1505 0:0 active_ext => bool; 1506 /// Internal address space command is active, a 1-bit boolean flag. 1507 1:1 active_int => bool; 1508 } 1509 } 1510 1511 /// Helpers for TRANSCFG register. 1512 /// 1513 /// Address space mode for TRANSCFG register. 1514 #[derive(Copy, Clone, Debug, PartialEq)] 1515 #[repr(u8)] 1516 pub(crate) enum AddressSpaceMode { 1517 /// The MMU forces all memory access to fail with a decode fault. 1518 Unmapped = 1, 1519 /// All input addresses map to the same output address (deprecated). 1520 Identity = 2, 1521 /// Translation tables interpreted according to AArch64 4kB granule specification. 1522 Aarch64_4K = 6, 1523 /// Translation tables interpreted according to AArch64 64kB granule specification. 1524 Aarch64_64K = 8, 1525 } 1526 1527 impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode { 1528 type Error = Error; 1529 1530 fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> { 1531 match val.get() { 1532 1 => Ok(AddressSpaceMode::Unmapped), 1533 2 => Ok(AddressSpaceMode::Identity), 1534 6 => Ok(AddressSpaceMode::Aarch64_4K), 1535 8 => Ok(AddressSpaceMode::Aarch64_64K), 1536 _ => Err(EINVAL), 1537 } 1538 } 1539 } 1540 1541 impl From<AddressSpaceMode> for Bounded<u64, 4> { 1542 fn from(mode: AddressSpaceMode) -> Self { 1543 Bounded::try_new(mode as u64).unwrap() 1544 } 1545 } 1546 1547 /// Input address range restriction for TRANSCFG register. 1548 #[derive(Copy, Clone, Debug, PartialEq)] 1549 #[repr(u8)] 1550 pub(crate) enum InaBits { 1551 /// Invalid VA range (reset value). 1552 Reset = 0, 1553 /// 48-bit VA range. 1554 Bits48 = 7, 1555 /// 47-bit VA range. 1556 Bits47 = 8, 1557 /// 46-bit VA range. 1558 Bits46 = 9, 1559 /// 45-bit VA range. 1560 Bits45 = 10, 1561 /// 44-bit VA range. 1562 Bits44 = 11, 1563 /// 43-bit VA range. 1564 Bits43 = 12, 1565 /// 42-bit VA range. 1566 Bits42 = 13, 1567 /// 41-bit VA range. 1568 Bits41 = 14, 1569 /// 40-bit VA range. 1570 Bits40 = 15, 1571 /// 39-bit VA range. 1572 Bits39 = 16, 1573 /// 38-bit VA range. 1574 Bits38 = 17, 1575 /// 37-bit VA range. 1576 Bits37 = 18, 1577 /// 36-bit VA range. 1578 Bits36 = 19, 1579 /// 35-bit VA range. 1580 Bits35 = 20, 1581 /// 34-bit VA range. 1582 Bits34 = 21, 1583 /// 33-bit VA range. 1584 Bits33 = 22, 1585 /// 32-bit VA range. 1586 Bits32 = 23, 1587 /// 31-bit VA range. 1588 Bits31 = 24, 1589 /// 30-bit VA range. 1590 Bits30 = 25, 1591 /// 29-bit VA range. 1592 Bits29 = 26, 1593 /// 28-bit VA range. 1594 Bits28 = 27, 1595 /// 27-bit VA range. 1596 Bits27 = 28, 1597 /// 26-bit VA range. 1598 Bits26 = 29, 1599 /// 25-bit VA range. 1600 Bits25 = 30, 1601 } 1602 1603 impl TryFrom<Bounded<u64, 5>> for InaBits { 1604 type Error = Error; 1605 1606 fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> { 1607 match val.get() { 1608 0 => Ok(InaBits::Reset), 1609 7 => Ok(InaBits::Bits48), 1610 8 => Ok(InaBits::Bits47), 1611 9 => Ok(InaBits::Bits46), 1612 10 => Ok(InaBits::Bits45), 1613 11 => Ok(InaBits::Bits44), 1614 12 => Ok(InaBits::Bits43), 1615 13 => Ok(InaBits::Bits42), 1616 14 => Ok(InaBits::Bits41), 1617 15 => Ok(InaBits::Bits40), 1618 16 => Ok(InaBits::Bits39), 1619 17 => Ok(InaBits::Bits38), 1620 18 => Ok(InaBits::Bits37), 1621 19 => Ok(InaBits::Bits36), 1622 20 => Ok(InaBits::Bits35), 1623 21 => Ok(InaBits::Bits34), 1624 22 => Ok(InaBits::Bits33), 1625 23 => Ok(InaBits::Bits32), 1626 24 => Ok(InaBits::Bits31), 1627 25 => Ok(InaBits::Bits30), 1628 26 => Ok(InaBits::Bits29), 1629 27 => Ok(InaBits::Bits28), 1630 28 => Ok(InaBits::Bits27), 1631 29 => Ok(InaBits::Bits26), 1632 30 => Ok(InaBits::Bits25), 1633 _ => Err(EINVAL), 1634 } 1635 } 1636 } 1637 1638 impl From<InaBits> for Bounded<u64, 5> { 1639 fn from(bits: InaBits) -> Self { 1640 Bounded::try_new(bits as u64).unwrap() 1641 } 1642 } 1643 1644 /// Translation table memory attributes for TRANSCFG register. 1645 #[derive(Copy, Clone, Debug, PartialEq)] 1646 #[repr(u8)] 1647 pub(crate) enum PtwMemattr { 1648 /// Invalid (reset value, not valid for enabled address space). 1649 Invalid = 0, 1650 /// Normal memory, inner/outer non-cacheable. 1651 NonCacheable = 1, 1652 /// Normal memory, inner/outer write-back cacheable. 1653 WriteBack = 2, 1654 } 1655 1656 impl TryFrom<Bounded<u64, 2>> for PtwMemattr { 1657 type Error = Error; 1658 1659 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1660 match val.get() { 1661 0 => Ok(PtwMemattr::Invalid), 1662 1 => Ok(PtwMemattr::NonCacheable), 1663 2 => Ok(PtwMemattr::WriteBack), 1664 _ => Err(EINVAL), 1665 } 1666 } 1667 } 1668 1669 impl From<PtwMemattr> for Bounded<u64, 2> { 1670 fn from(attr: PtwMemattr) -> Self { 1671 Bounded::try_new(attr as u64).unwrap() 1672 } 1673 } 1674 1675 /// Translation table memory shareability for TRANSCFG register. 1676 #[derive(Copy, Clone, Debug, PartialEq)] 1677 #[repr(u8)] 1678 #[allow(clippy::enum_variant_names)] 1679 pub(crate) enum PtwShareability { 1680 /// Non-shareable. 1681 NonShareable = 0, 1682 /// Outer shareable. 1683 OuterShareable = 2, 1684 /// Inner shareable. 1685 InnerShareable = 3, 1686 } 1687 1688 impl TryFrom<Bounded<u64, 2>> for PtwShareability { 1689 type Error = Error; 1690 1691 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1692 match val.get() { 1693 0 => Ok(PtwShareability::NonShareable), 1694 2 => Ok(PtwShareability::OuterShareable), 1695 3 => Ok(PtwShareability::InnerShareable), 1696 _ => Err(EINVAL), 1697 } 1698 } 1699 } 1700 1701 impl From<PtwShareability> for Bounded<u64, 2> { 1702 fn from(sh: PtwShareability) -> Self { 1703 Bounded::try_new(sh as u64).unwrap() 1704 } 1705 } 1706 1707 register! { 1708 /// Translation configuration and control. 1709 pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { 1710 /// Address space mode. 1711 3:0 mode ?=> AddressSpaceMode; 1712 /// Address input restriction. 1713 10:6 ina_bits ?=> InaBits; 1714 /// Address output restriction. 1715 18:14 outa_bits; 1716 /// Translation table concatenation enable, a 1-bit boolean flag. 1717 22:22 sl_concat_en => bool; 1718 /// Translation table memory attributes. 1719 25:24 ptw_memattr ?=> PtwMemattr; 1720 /// Translation table memory shareability. 1721 29:28 ptw_sh ?=> PtwShareability; 1722 /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. 1723 30:30 r_allocate => bool; 1724 /// Disable hierarchical access permissions. 1725 33:33 disable_hier_ap => bool; 1726 /// Disable access fault checking. 1727 34:34 disable_af_fault => bool; 1728 /// Disable execution on all writable pages. 1729 35:35 wxn => bool; 1730 /// Enable execution on readable pages. 1731 36:36 xreadable => bool; 1732 /// Page-based hardware attributes for translation table walks. 1733 63:60 ptw_pbha; 1734 } 1735 1736 // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two 1737 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1738 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1739 pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { 1740 31:0 value; 1741 } 1742 1743 pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { 1744 31:0 value; 1745 } 1746 1747 /// Extra fault information for each address space. Read only. 1748 pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { 1749 31:0 value; 1750 } 1751 1752 pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { 1753 31:0 value; 1754 } 1755 } 1756 } 1757 } 1758 1759 /// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. 1760 pub(crate) mod doorbell_block { 1761 use kernel::register; 1762 1763 /// Number of doorbells available. 1764 pub(crate) const NUM_DOORBELLS: usize = 64; 1765 1766 /// Doorbell block stride (64KiB). 1767 /// 1768 /// Each block occupies a full page, allowing it to be mapped 1769 /// separately into a virtual address space. 1770 const STRIDE: usize = 0x10000; 1771 1772 register! { 1773 /// Doorbell request register. Write-only. 1774 pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { 1775 /// Doorbell set. Writing 1 triggers the doorbell. 1776 0:0 ring => bool; 1777 } 1778 } 1779 } 1780