1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 3 //! # Definitions 4 //! 5 //! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) 6 //! from the command stream. 7 //! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular 8 //! job or sequence of jobs. The instructions exist in one or more command buffers. 9 //! - **CSF**: Command Stream Frontend - The interface and implementation for job submission 10 //! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS 11 //! interfaces. 12 //! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple 13 //! CSGs, and each CSG contains multiple CSs. 14 //! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the 15 //! iterator control aspects. Implements the CSF in conjunction with the MCU. 16 //! - **GLB**: Global - Prefix for global interface registers that control operations common to 17 //! all CSs. 18 //! - **JASID**: Job Address Space ID - Identifies the address space for a job. 19 //! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream 20 //! hardware. 21 //! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. 22 23 // We don't expect that all the registers and fields will be used, even in the 24 // future. 25 // 26 // Nevertheless, it is useful to have most of them defined, like the C driver 27 // does. 28 #![expect(dead_code)] 29 30 /// Combine two 32-bit values into a single 64-bit value. 31 pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { 32 (u64::from(lo)) | ((u64::from(hi)) << 32) 33 } 34 35 /// Read a logical 64-bit value from split 32-bit registers without tearing. 36 pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { 37 loop { 38 let hi1 = hi_read(); 39 let lo = lo_read(); 40 let hi2 = hi_read(); 41 42 if hi1 == hi2 { 43 return join_u64(lo, hi1); 44 } 45 } 46 } 47 48 pub(crate) use mmu_control::mmu_as_control::MAX_AS; 49 50 /// These registers correspond to the GPU_CONTROL register page. 51 /// They are involved in GPU configuration and control. 52 pub(crate) mod gpu_control { 53 use kernel::{ 54 num::Bounded, 55 prelude::*, 56 register, 57 uapi, // 58 }; 59 60 use crate::driver::TyrRegisters; 61 62 register! { 63 base: TyrRegisters; 64 65 /// GPU identification register. 66 pub(crate) GPU_ID(u32) @ 0x0 { 67 /// Status of the GPU release. 68 3:0 ver_status; 69 /// Minor release version number. 70 11:4 ver_minor; 71 /// Major release version number. 72 15:12 ver_major; 73 /// Product identifier. 74 19:16 prod_major; 75 /// Architecture patch revision. 76 23:20 arch_rev; 77 /// Architecture minor revision. 78 27:24 arch_minor; 79 /// Architecture major revision. 80 31:28 arch_major; 81 } 82 83 /// Level 2 cache features register. 84 pub(crate) L2_FEATURES(u32) @ 0x4 { 85 /// Cache line size. 86 7:0 line_size; 87 /// Cache associativity. 88 15:8 associativity; 89 /// Cache slice size. 90 23:16 cache_size; 91 /// External bus width. 92 31:24 bus_width; 93 } 94 95 /// Shader core features. 96 pub(crate) CORE_FEATURES(u32) @ 0x8 { 97 /// Shader core variant. 98 7:0 core_variant; 99 } 100 101 /// Tiler features. 102 pub(crate) TILER_FEATURES(u32) @ 0xc { 103 /// Log of the tiler's bin size. 104 5:0 bin_size; 105 /// Maximum number of active levels. 106 11:8 max_levels; 107 } 108 109 /// Memory system features. 110 pub(crate) MEM_FEATURES(u32) @ 0x10 { 111 0:0 coherent_core_group => bool; 112 1:1 coherent_super_group => bool; 113 11:8 l2_slices; 114 } 115 116 /// Memory management unit features. 117 pub(crate) MMU_FEATURES(u32) @ 0x14 { 118 /// Number of bits supported in virtual addresses. 119 7:0 va_bits; 120 /// Number of bits supported in physical addresses. 121 15:8 pa_bits; 122 } 123 124 /// Address spaces present. 125 pub(crate) AS_PRESENT(u32) @ 0x18 { 126 31:0 present; 127 } 128 129 /// CSF version information. 130 pub(crate) CSF_ID(u32) @ 0x1c { 131 /// MCU revision ID. 132 3:0 mcu_rev; 133 /// MCU minor revision number. 134 9:4 mcu_minor; 135 /// MCU major revision number. 136 15:10 mcu_major; 137 /// CSHW revision ID. 138 19:16 cshw_rev; 139 /// CSHW minor revision number. 140 25:20 cshw_minor; 141 /// CSHW major revision number. 142 31:26 cshw_major; 143 } 144 145 /// IRQ sources raw status. 146 /// Writing to this register forces bits on, but does not clear them. 147 pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { 148 /// A GPU fault has occurred, a 1-bit boolean flag. 149 0:0 gpu_fault => bool; 150 /// A GPU fault has occurred, a 1-bit boolean flag. 151 1:1 gpu_protected_fault => bool; 152 /// Reset has completed, a 1-bit boolean flag. 153 8:8 reset_completed => bool; 154 /// Set when a single power domain has powered up or down, a 1-bit boolean flag. 155 9:9 power_changed_single => bool; 156 /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. 157 10:10 power_changed_all => bool; 158 /// Set when cache cleaning has completed, a 1-bit boolean flag. 159 17:17 clean_caches_completed => bool; 160 /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. 161 18:18 doorbell_mirror => bool; 162 /// MCU requires attention, a 1-bit boolean flag. 163 19:19 mcu_status => bool; 164 } 165 166 /// IRQ sources to clear. Write only. 167 pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { 168 /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. 169 0:0 gpu_fault => bool; 170 /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 171 1:1 gpu_protected_fault => bool; 172 /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. 173 8:8 reset_completed => bool; 174 /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 175 9:9 power_changed_single => bool; 176 /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 177 10:10 power_changed_all => bool; 178 /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 179 17:17 clean_caches_completed => bool; 180 /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. 181 19:19 mcu_status => bool; 182 } 183 184 /// IRQ sources enabled. 185 pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { 186 /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. 187 0:0 gpu_fault => bool; 188 /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 189 1:1 gpu_protected_fault => bool; 190 /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. 191 8:8 reset_completed => bool; 192 /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 193 9:9 power_changed_single => bool; 194 /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 195 10:10 power_changed_all => bool; 196 /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 197 17:17 clean_caches_completed => bool; 198 /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. 199 18:18 doorbell_mirror => bool; 200 /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. 201 19:19 mcu_status => bool; 202 } 203 204 /// IRQ status for enabled sources. Read only. 205 pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { 206 /// GPU_FAULT interrupt status, a 1-bit boolean flag. 207 0:0 gpu_fault => bool; 208 /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. 209 1:1 gpu_protected_fault => bool; 210 /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. 211 8:8 reset_completed => bool; 212 /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. 213 9:9 power_changed_single => bool; 214 /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. 215 10:10 power_changed_all => bool; 216 /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. 217 17:17 clean_caches_completed => bool; 218 /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. 219 18:18 doorbell_mirror => bool; 220 /// MCU_STATUS interrupt status, a 1-bit boolean flag. 221 19:19 mcu_status => bool; 222 } 223 } 224 225 /// Helpers for GPU_COMMAND Register 226 #[derive(Copy, Clone, Debug, PartialEq)] 227 #[repr(u8)] 228 pub(crate) enum GpuCommand { 229 /// No operation. This is the default value. 230 Nop = 0, 231 /// Reset the GPU. 232 Reset = 1, 233 /// Flush caches. 234 FlushCaches = 4, 235 /// Clear GPU faults. 236 ClearFault = 7, 237 } 238 239 impl TryFrom<Bounded<u32, 8>> for GpuCommand { 240 type Error = Error; 241 242 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 243 match val.get() { 244 0 => Ok(GpuCommand::Nop), 245 1 => Ok(GpuCommand::Reset), 246 4 => Ok(GpuCommand::FlushCaches), 247 7 => Ok(GpuCommand::ClearFault), 248 _ => Err(EINVAL), 249 } 250 } 251 } 252 253 impl From<GpuCommand> for Bounded<u32, 8> { 254 fn from(cmd: GpuCommand) -> Self { 255 (cmd as u8).into() 256 } 257 } 258 259 /// Reset mode for [`GPU_COMMAND::reset()`]. 260 #[derive(Copy, Clone, Debug, PartialEq)] 261 #[repr(u8)] 262 pub(crate) enum ResetMode { 263 /// Stop all external bus interfaces, then reset the entire GPU. 264 SoftReset = 1, 265 /// Force a full GPU reset. 266 HardReset = 2, 267 } 268 269 impl TryFrom<Bounded<u32, 4>> for ResetMode { 270 type Error = Error; 271 272 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 273 match val.get() { 274 1 => Ok(ResetMode::SoftReset), 275 2 => Ok(ResetMode::HardReset), 276 _ => Err(EINVAL), 277 } 278 } 279 } 280 281 impl From<ResetMode> for Bounded<u32, 4> { 282 fn from(mode: ResetMode) -> Self { 283 Bounded::try_new(mode as u32).unwrap() 284 } 285 } 286 287 /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. 288 #[derive(Copy, Clone, Debug, PartialEq)] 289 #[repr(u8)] 290 pub(crate) enum FlushMode { 291 /// No flush. 292 None = 0, 293 /// Clean the caches. 294 Clean = 1, 295 /// Invalidate the caches. 296 Invalidate = 2, 297 /// Clean and invalidate the caches. 298 CleanInvalidate = 3, 299 } 300 301 impl TryFrom<Bounded<u32, 4>> for FlushMode { 302 type Error = Error; 303 304 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 305 match val.get() { 306 0 => Ok(FlushMode::None), 307 1 => Ok(FlushMode::Clean), 308 2 => Ok(FlushMode::Invalidate), 309 3 => Ok(FlushMode::CleanInvalidate), 310 _ => Err(EINVAL), 311 } 312 } 313 } 314 315 impl From<FlushMode> for Bounded<u32, 4> { 316 fn from(mode: FlushMode) -> Self { 317 Bounded::try_new(mode as u32).unwrap() 318 } 319 } 320 321 register! { 322 base: TyrRegisters; 323 324 /// GPU command register. 325 /// 326 /// Use the constructor methods to create commands: 327 /// - [`GPU_COMMAND::nop()`] 328 /// - [`GPU_COMMAND::reset()`] 329 /// - [`GPU_COMMAND::flush_caches()`] 330 /// - [`GPU_COMMAND::clear_fault()`] 331 pub(crate) GPU_COMMAND (u32) @ 0x30 { 332 7:0 command ?=> GpuCommand; 333 } 334 /// Internal alias for GPU_COMMAND in reset mode. 335 /// Use [`GPU_COMMAND::reset()`] instead. 336 GPU_COMMAND_RESET (u32) => GPU_COMMAND { 337 7:0 command ?=> GpuCommand; 338 11:8 reset_mode ?=> ResetMode; 339 } 340 341 /// Internal alias for GPU_COMMAND in cache flush mode. 342 /// Use [`GPU_COMMAND::flush_caches()`] instead. 343 GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { 344 7:0 command ?=> GpuCommand; 345 /// L2 cache flush mode. 346 11:8 l2_flush ?=> FlushMode; 347 /// Shader core load/store cache flush mode. 348 15:12 lsc_flush ?=> FlushMode; 349 /// Shader core other caches flush mode. 350 19:16 other_flush ?=> FlushMode; 351 } 352 } 353 354 impl GPU_COMMAND { 355 /// Create a NOP command. 356 pub(crate) fn nop() -> Self { 357 Self::zeroed() 358 } 359 360 /// Create a reset command with the specified reset mode. 361 pub(crate) fn reset(mode: ResetMode) -> Self { 362 Self::from_raw( 363 GPU_COMMAND_RESET::zeroed() 364 .with_command(GpuCommand::Reset) 365 .with_reset_mode(mode) 366 .into_raw(), 367 ) 368 } 369 370 /// Create a cache flush command with the specified flush modes. 371 pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { 372 Self::from_raw( 373 GPU_COMMAND_FLUSH::zeroed() 374 .with_command(GpuCommand::FlushCaches) 375 .with_l2_flush(l2) 376 .with_lsc_flush(lsc) 377 .with_other_flush(other) 378 .into_raw(), 379 ) 380 } 381 382 /// Create a clear fault command. 383 pub(crate) fn clear_fault() -> Self { 384 Self::zeroed().with_command(GpuCommand::ClearFault) 385 } 386 } 387 388 register! { 389 base: TyrRegisters; 390 391 /// GPU status register. Read only. 392 pub(crate) GPU_STATUS(u32) @ 0x34 { 393 /// GPU active, a 1-bit boolean flag. 394 0:0 gpu_active => bool; 395 /// Power manager active, a 1-bit boolean flag 396 1:1 pwr_active => bool; 397 /// Page fault active, a 1-bit boolean flag. 398 4:4 page_fault => bool; 399 /// Protected mode active, a 1-bit boolean flag. 400 7:7 protected_mode_active => bool; 401 /// Debug mode active, a 1-bit boolean flag. 402 8:8 gpu_dbg_enabled => bool; 403 } 404 } 405 406 #[derive(Copy, Clone, Debug, PartialEq)] 407 #[repr(u8)] 408 pub(crate) enum ExceptionType { 409 /// Exception type: No error. 410 Ok = 0x00, 411 /// Exception type: GPU external bus error. 412 GpuBusFault = 0x80, 413 /// Exception type: GPU shareability error. 414 GpuShareabilityFault = 0x88, 415 /// Exception type: System shareability error. 416 SystemShareabilityFault = 0x89, 417 /// Exception type: GPU cacheability error. 418 GpuCacheabilityFault = 0x8A, 419 } 420 421 impl TryFrom<Bounded<u32, 8>> for ExceptionType { 422 type Error = Error; 423 424 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 425 match val.get() { 426 0x00 => Ok(ExceptionType::Ok), 427 0x80 => Ok(ExceptionType::GpuBusFault), 428 0x88 => Ok(ExceptionType::GpuShareabilityFault), 429 0x89 => Ok(ExceptionType::SystemShareabilityFault), 430 0x8A => Ok(ExceptionType::GpuCacheabilityFault), 431 _ => Err(EINVAL), 432 } 433 } 434 } 435 436 impl From<ExceptionType> for Bounded<u32, 8> { 437 fn from(exc: ExceptionType) -> Self { 438 (exc as u8).into() 439 } 440 } 441 442 #[derive(Copy, Clone, Debug, PartialEq)] 443 #[repr(u8)] 444 pub(crate) enum AccessType { 445 /// Access type: An atomic (read/write) transaction. 446 Atomic = 0, 447 /// Access type: An execute transaction. 448 Execute = 1, 449 /// Access type: A read transaction. 450 Read = 2, 451 /// Access type: A write transaction. 452 Write = 3, 453 } 454 455 impl From<Bounded<u32, 2>> for AccessType { 456 fn from(val: Bounded<u32, 2>) -> Self { 457 match val.get() { 458 0 => AccessType::Atomic, 459 1 => AccessType::Execute, 460 2 => AccessType::Read, 461 3 => AccessType::Write, 462 _ => unreachable!(), 463 } 464 } 465 } 466 467 impl From<AccessType> for Bounded<u32, 2> { 468 fn from(access: AccessType) -> Self { 469 Bounded::try_new(access as u32).unwrap() 470 } 471 } 472 473 register! { 474 base: TyrRegisters; 475 476 /// GPU fault status register. Read only. 477 pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { 478 /// Exception type. 479 7:0 exception_type ?=> ExceptionType; 480 /// Access type. 481 9:8 access_type => AccessType; 482 /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. 483 10:10 address_valid => bool; 484 /// The JASID field is valid, a 1-bit boolean flag. 485 11:11 jasid_valid => bool; 486 /// JASID of the fault, if known. 487 15:12 jasid; 488 /// ID of the source that triggered the fault. 489 31:16 source_id; 490 } 491 492 /// GPU fault address. Read only. 493 /// Once a fault is reported, it must be manually cleared by issuing a 494 /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU 495 /// faults will be reported until the previous fault has been cleared. 496 pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { 497 31:0 pointer; 498 } 499 500 pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { 501 31:0 pointer; 502 } 503 504 /// Level 2 cache configuration. 505 pub(crate) L2_CONFIG(u32) @ 0x48 { 506 /// Requested cache size. 507 23:16 cache_size; 508 /// Requested hash function index. 509 31:24 hash_function; 510 } 511 512 /// Global time stamp offset. 513 pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { 514 31:0 offset; 515 } 516 517 pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { 518 31:0 offset; 519 } 520 521 /// GPU cycle counter. Read only. 522 pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { 523 31:0 count; 524 } 525 526 pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { 527 31:0 count; 528 } 529 530 /// Global time stamp. Read only. 531 pub(crate) TIMESTAMP_LO(u32) @ 0x98 { 532 31:0 timestamp; 533 } 534 535 pub(crate) TIMESTAMP_HI(u32) @ 0x9c { 536 31:0 timestamp; 537 } 538 539 /// Maximum number of threads per core. Read only constant. 540 pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { 541 31:0 threads; 542 } 543 544 /// Maximum number of threads per workgroup. Read only constant. 545 pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { 546 31:0 threads; 547 } 548 549 /// Maximum number of threads per barrier. Read only constant. 550 pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { 551 31:0 threads; 552 } 553 554 /// Thread features. Read only constant. 555 pub(crate) THREAD_FEATURES(u32) @ 0xac { 556 /// Total number of registers per core. 557 21:0 max_registers; 558 /// Implementation technology type. 559 23:22 implementation_technology; 560 /// Maximum number of compute tasks waiting. 561 31:24 max_task_queue; 562 } 563 564 /// Support flags for compressed texture formats. Read only constant. 565 /// 566 /// A bitmap where each bit indicates support for a specific compressed texture format. 567 /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). 568 pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { 569 31:0 supported_formats; 570 } 571 572 /// Shader core present bitmap. Read only constant. 573 pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { 574 31:0 value; 575 } 576 577 pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { 578 31:0 value; 579 } 580 581 /// Tiler present bitmap. Read only constant. 582 pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { 583 31:0 present; 584 } 585 586 pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { 587 31:0 present; 588 } 589 590 /// L2 cache present bitmap. Read only constant. 591 pub(crate) L2_PRESENT_LO(u32) @ 0x120 { 592 31:0 present; 593 } 594 595 pub(crate) L2_PRESENT_HI(u32) @ 0x124 { 596 31:0 present; 597 } 598 599 /// Shader core ready bitmap. Read only. 600 pub(crate) SHADER_READY_LO(u32) @ 0x140 { 601 31:0 ready; 602 } 603 604 pub(crate) SHADER_READY_HI(u32) @ 0x144 { 605 31:0 ready; 606 } 607 608 /// Tiler ready bitmap. Read only. 609 pub(crate) TILER_READY_LO(u32) @ 0x150 { 610 31:0 ready; 611 } 612 613 pub(crate) TILER_READY_HI(u32) @ 0x154 { 614 31:0 ready; 615 } 616 617 /// L2 ready bitmap. Read only. 618 pub(crate) L2_READY_LO(u32) @ 0x160 { 619 31:0 ready; 620 } 621 622 pub(crate) L2_READY_HI(u32) @ 0x164 { 623 31:0 ready; 624 } 625 626 /// Shader core power up bitmap. 627 pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { 628 31:0 request; 629 } 630 631 pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { 632 31:0 request; 633 } 634 635 /// Tiler power up bitmap. 636 pub(crate) TILER_PWRON_LO(u32) @ 0x190 { 637 31:0 request; 638 } 639 640 pub(crate) TILER_PWRON_HI(u32) @ 0x194 { 641 31:0 request; 642 } 643 644 /// L2 power up bitmap. 645 pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { 646 31:0 request; 647 } 648 649 pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { 650 31:0 request; 651 } 652 653 /// Shader core power down bitmap. 654 pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { 655 31:0 request; 656 } 657 658 pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { 659 31:0 request; 660 } 661 662 /// Tiler power down bitmap. 663 pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { 664 31:0 request; 665 } 666 667 pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { 668 31:0 request; 669 } 670 671 /// L2 power down bitmap. 672 pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { 673 31:0 request; 674 } 675 676 pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { 677 31:0 request; 678 } 679 680 /// Shader core power transition bitmap. Read-only. 681 pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { 682 31:0 changing; 683 } 684 685 pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { 686 31:0 changing; 687 } 688 689 /// Tiler power transition bitmap. Read-only. 690 pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { 691 31:0 changing; 692 } 693 694 pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { 695 31:0 changing; 696 } 697 698 /// L2 power transition bitmap. Read-only. 699 pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { 700 31:0 changing; 701 } 702 703 pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { 704 31:0 changing; 705 } 706 707 /// Shader core active bitmap. Read-only. 708 pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { 709 31:0 active; 710 } 711 712 pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { 713 31:0 active; 714 } 715 716 /// Tiler active bitmap. Read-only. 717 pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { 718 31:0 active; 719 } 720 721 pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { 722 31:0 active; 723 } 724 725 /// L2 active bitmap. Read-only. 726 pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { 727 31:0 active; 728 } 729 730 pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { 731 31:0 active; 732 } 733 734 /// Revision ID. Read only constant. 735 pub(crate) REVIDR(u32) @ 0x280 { 736 31:0 revision; 737 } 738 739 /// Coherency features present. Read only constant. 740 /// Supported protocols on the interconnect between the GPU and the 741 /// system into which it is integrated. 742 pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { 743 /// ACE-Lite protocol supported, a 1-bit boolean flag. 744 0:0 ace_lite => bool; 745 /// ACE protocol supported, a 1-bit boolean flag. 746 1:1 ace => bool; 747 } 748 } 749 750 #[derive(Copy, Clone, Debug, PartialEq)] 751 #[repr(u8)] 752 pub(crate) enum CoherencyMode { 753 /// ACE-Lite coherency protocol. 754 AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, 755 /// ACE coherency protocol. 756 Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, 757 /// No coherency protocol. 758 None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, 759 } 760 761 impl TryFrom<Bounded<u32, 32>> for CoherencyMode { 762 type Error = Error; 763 764 fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> { 765 match val.get() { 766 0 => Ok(CoherencyMode::AceLite), 767 1 => Ok(CoherencyMode::Ace), 768 31 => Ok(CoherencyMode::None), 769 _ => Err(EINVAL), 770 } 771 } 772 } 773 774 impl From<CoherencyMode> for Bounded<u32, 32> { 775 fn from(mode: CoherencyMode) -> Self { 776 (mode as u8).into() 777 } 778 } 779 780 register! { 781 base: TyrRegisters; 782 783 /// Coherency enable. An index of which coherency protocols should be used. 784 /// This register only selects the protocol for coherency messages on the 785 /// interconnect. This is not to enable or disable coherency controlled by MMU. 786 pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { 787 31:0 l2_cache_protocol_select ?=> CoherencyMode; 788 } 789 } 790 791 /// Helpers for MCU_CONTROL register 792 #[derive(Copy, Clone, Debug, PartialEq)] 793 #[repr(u8)] 794 pub(crate) enum McuControlMode { 795 /// Disable the MCU. 796 Disable = 0, 797 /// Enable the MCU. 798 Enable = 1, 799 /// Enable the MCU to execute and automatically reboot after a fast reset. 800 Auto = 2, 801 } 802 803 impl TryFrom<Bounded<u32, 2>> for McuControlMode { 804 type Error = Error; 805 806 fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> { 807 match val.get() { 808 0 => Ok(McuControlMode::Disable), 809 1 => Ok(McuControlMode::Enable), 810 2 => Ok(McuControlMode::Auto), 811 _ => Err(EINVAL), 812 } 813 } 814 } 815 816 impl From<McuControlMode> for Bounded<u32, 2> { 817 fn from(mode: McuControlMode) -> Self { 818 Bounded::try_new(mode as u32).unwrap() 819 } 820 } 821 822 register! { 823 base: TyrRegisters; 824 825 /// MCU control. 826 pub(crate) MCU_CONTROL(u32) @ 0x700 { 827 /// Request MCU state change. 828 1:0 req ?=> McuControlMode; 829 } 830 } 831 832 /// Helpers for MCU_STATUS register 833 #[derive(Copy, Clone, Debug, PartialEq)] 834 #[repr(u8)] 835 pub(crate) enum McuStatus { 836 /// MCU is disabled. 837 Disabled = 0, 838 /// MCU is enabled. 839 Enabled = 1, 840 /// The MCU has halted by itself in an orderly manner to enable the core group to be 841 /// powered down. 842 Halt = 2, 843 /// The MCU has encountered an error that prevents it from continuing. 844 Fatal = 3, 845 } 846 847 impl From<Bounded<u32, 2>> for McuStatus { 848 fn from(val: Bounded<u32, 2>) -> Self { 849 match val.get() { 850 0 => McuStatus::Disabled, 851 1 => McuStatus::Enabled, 852 2 => McuStatus::Halt, 853 3 => McuStatus::Fatal, 854 _ => unreachable!(), 855 } 856 } 857 } 858 859 impl From<McuStatus> for Bounded<u32, 2> { 860 fn from(status: McuStatus) -> Self { 861 Bounded::try_new(status as u32).unwrap() 862 } 863 } 864 865 register! { 866 base: TyrRegisters; 867 868 /// MCU status. Read only. 869 pub(crate) MCU_STATUS(u32) @ 0x704 { 870 /// Read current state of MCU. 871 1:0 value => McuStatus; 872 } 873 } 874 } 875 876 /// These registers correspond to the JOB_CONTROL register page. 877 /// They are involved in communication between the firmware running on the MCU and the host. 878 pub(crate) mod job_control { 879 use kernel::register; 880 881 use crate::driver::TyrRegisters; 882 883 register! { 884 base: TyrRegisters; 885 886 /// Raw status of job interrupts. 887 /// 888 /// Write to this register to trigger these interrupts. 889 /// Writing a 1 to a bit forces that bit on. 890 pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { 891 /// CSG request. These bits indicate that CSGn requires attention from the host. 892 30:0 csg; 893 /// GLB request. Indicates that the GLB interface requires attention from the host. 894 31:31 glb => bool; 895 } 896 897 /// Clear job interrupts. Write only. 898 /// 899 /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. 900 pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { 901 /// Clear CSG request interrupts. 902 30:0 csg; 903 /// Clear GLB request interrupt. 904 31:31 glb => bool; 905 } 906 907 /// Mask for job interrupts. 908 /// 909 /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. 910 pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { 911 /// Enable CSG request interrupts. 912 30:0 csg; 913 /// Enable GLB request interrupt. 914 31:31 glb => bool; 915 } 916 917 /// Active job interrupts. Read only. 918 /// 919 /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and 920 /// [`JOB_IRQ_MASK`]. 921 pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { 922 /// CSG request interrupt status. 923 30:0 csg; 924 /// GLB request interrupt status. 925 31:31 glb => bool; 926 } 927 } 928 } 929 930 /// These registers correspond to the MMU_CONTROL register page. 931 /// They are involved in MMU configuration and control. 932 pub(crate) mod mmu_control { 933 use kernel::register; 934 935 use crate::driver::TyrRegisters; 936 937 register! { 938 base: TyrRegisters; 939 940 /// IRQ sources raw status. 941 /// 942 /// This register contains the raw unmasked interrupt sources for MMU status and exception 943 /// handling. 944 /// 945 /// Writing to this register forces bits on. 946 /// Use [`IRQ_CLEAR`] to clear interrupts. 947 pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { 948 /// Page fault for address spaces. 949 15:0 page_fault; 950 /// Command completed in address spaces. 951 31:16 command_completed; 952 } 953 954 /// IRQ sources to clear. 955 /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. 956 pub(crate) IRQ_CLEAR(u32) @ 0x2004 { 957 /// Clear the PAGE_FAULT interrupt. 958 15:0 page_fault; 959 /// Clear the COMMAND_COMPLETED interrupt. 960 31:16 command_completed; 961 } 962 963 /// IRQ sources enabled. 964 /// 965 /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. 966 pub(crate) IRQ_MASK(u32) @ 0x2008 { 967 /// Enable the PAGE_FAULT interrupt. 968 15:0 page_fault; 969 /// Enable the COMMAND_COMPLETED interrupt. 970 31:16 command_completed; 971 } 972 973 /// IRQ status for enabled sources. Read only. 974 /// 975 /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. 976 pub(crate) IRQ_STATUS(u32) @ 0x200c { 977 /// PAGE_FAULT interrupt status. 978 15:0 page_fault; 979 /// COMMAND_COMPLETED interrupt status. 980 31:16 command_completed; 981 } 982 } 983 984 /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. 985 /// 986 /// This array contains 16 instances of the MMU_AS_CONTROL register page. 987 pub(crate) mod mmu_as_control { 988 use kernel::{ 989 num::Bounded, 990 prelude::*, 991 register, // 992 }; 993 use pin_init::Zeroable; 994 995 use crate::driver::TyrRegisters; 996 997 /// Maximum number of hardware address space slots. 998 /// The actual number of slots available is usually lower. 999 pub(crate) const MAX_AS: usize = 16; 1000 1001 /// Address space register stride. The elements in the array are spaced 64B apart. 1002 const STRIDE: usize = 0x40; 1003 1004 register! { 1005 base: TyrRegisters; 1006 1007 /// Translation table base address. A 64-bit pointer. 1008 /// 1009 /// This field contains the address of the top level of a translation table structure. 1010 /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. 1011 pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { 1012 /// Base address of the translation table. 1013 63:0 base; 1014 } 1015 1016 // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two 1017 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1018 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1019 pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { 1020 31:0 value; 1021 } 1022 1023 pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { 1024 31:0 value; 1025 } 1026 } 1027 1028 /// Helpers for MEMATTR Register. 1029 1030 #[derive(Copy, Clone, Debug, PartialEq)] 1031 #[repr(u8)] 1032 pub(crate) enum AllocPolicySelect { 1033 /// Ignore ALLOC_R/ALLOC_W fields. 1034 Impl = 2, 1035 /// Use ALLOC_R/ALLOC_W fields for allocation policy. 1036 Alloc = 3, 1037 } 1038 1039 impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect { 1040 type Error = Error; 1041 1042 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1043 match val.get() { 1044 2 => Ok(Self::Impl), 1045 3 => Ok(Self::Alloc), 1046 _ => Err(EINVAL), 1047 } 1048 } 1049 } 1050 1051 impl From<AllocPolicySelect> for Bounded<u8, 2> { 1052 fn from(val: AllocPolicySelect) -> Self { 1053 Bounded::try_new(val as u8).unwrap() 1054 } 1055 } 1056 1057 /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. 1058 /// 1059 /// The hardware spec defines different interpretations of these values depending on 1060 /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation 1061 /// tables (all input addresses map to the same output address); it is deprecated and not 1062 /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. 1063 #[derive(Copy, Clone, Debug, PartialEq)] 1064 #[repr(u8)] 1065 pub(crate) enum Coherency { 1066 /// Midgard inner domain coherency. 1067 /// 1068 /// Most flexible mode - can map non-coherent, internally coherent, and system/IO 1069 /// coherent memory. Used for non-cacheable memory in MAIR conversion. 1070 MidgardInnerDomain = 0, 1071 /// CPU inner domain coherency. 1072 /// 1073 /// Can map non-coherent and system/IO coherent memory. Used for write-back 1074 /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. 1075 CpuInnerDomain = 1, 1076 /// CPU inner domain with shader coherency. 1077 /// 1078 /// Can map internally coherent and system/IO coherent memory. Used for 1079 /// GPU-internal shared buffers requiring shader coherency. 1080 CpuInnerDomainShaderCoh = 2, 1081 } 1082 1083 impl TryFrom<Bounded<u8, 2>> for Coherency { 1084 type Error = Error; 1085 1086 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1087 match val.get() { 1088 0 => Ok(Self::MidgardInnerDomain), 1089 1 => Ok(Self::CpuInnerDomain), 1090 2 => Ok(Self::CpuInnerDomainShaderCoh), 1091 _ => Err(EINVAL), 1092 } 1093 } 1094 } 1095 1096 impl From<Coherency> for Bounded<u8, 2> { 1097 fn from(val: Coherency) -> Self { 1098 Bounded::try_new(val as u8).unwrap() 1099 } 1100 } 1101 1102 #[derive(Copy, Clone, Debug, PartialEq)] 1103 #[repr(u8)] 1104 pub(crate) enum MemoryType { 1105 /// Normal memory (shared). 1106 Shared = 0, 1107 /// Normal memory, inner/outer non-cacheable. 1108 NonCacheable = 1, 1109 /// Normal memory, inner/outer write-back cacheable. 1110 WriteBack = 2, 1111 /// Triggers MEMORY_ATTRIBUTE_FAULT. 1112 Fault = 3, 1113 } 1114 1115 impl From<Bounded<u8, 2>> for MemoryType { 1116 fn from(val: Bounded<u8, 2>) -> Self { 1117 match val.get() { 1118 0 => Self::Shared, 1119 1 => Self::NonCacheable, 1120 2 => Self::WriteBack, 1121 3 => Self::Fault, 1122 _ => unreachable!(), 1123 } 1124 } 1125 } 1126 1127 impl From<MemoryType> for Bounded<u8, 2> { 1128 fn from(val: MemoryType) -> Self { 1129 Bounded::try_new(val as u8).unwrap() 1130 } 1131 } 1132 1133 register! { 1134 base: TyrRegisters; 1135 1136 /// Stage 1 memory attributes (8-bit bitfield). 1137 /// 1138 /// This is not an actual register, but a bitfield definition used by the MEMATTR 1139 /// register. Each of the 8 bytes in MEMATTR follows this layout. 1140 MMU_MEMATTR_STAGE1(u8) @ 0x0 { 1141 /// Inner cache write allocation policy. 1142 0:0 alloc_w => bool; 1143 /// Inner cache read allocation policy. 1144 1:1 alloc_r => bool; 1145 /// Inner allocation policy select. 1146 3:2 alloc_sel ?=> AllocPolicySelect; 1147 /// Coherency policy. 1148 5:4 coherency ?=> Coherency; 1149 /// Memory type. 1150 7:6 memory_type => MemoryType; 1151 } 1152 } 1153 1154 impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 { 1155 type Error = Error; 1156 1157 fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> { 1158 Ok(Self::from_raw(val.get() as u8)) 1159 } 1160 } 1161 1162 impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> { 1163 fn from(val: MMU_MEMATTR_STAGE1) -> Self { 1164 Bounded::try_new(u64::from(val.into_raw())).unwrap() 1165 } 1166 } 1167 1168 register! { 1169 base: TyrRegisters; 1170 1171 /// Memory attributes. 1172 /// 1173 /// Each address space can configure up to 8 different memory attribute profiles. 1174 /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. 1175 pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { 1176 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; 1177 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; 1178 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; 1179 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; 1180 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; 1181 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; 1182 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; 1183 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; 1184 } 1185 1186 // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two 1187 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1188 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1189 pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { 1190 31:0 value; 1191 } 1192 1193 pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { 1194 31:0 value; 1195 } 1196 } 1197 1198 impl MEMATTR { 1199 /// Outer cache-policy nibble indicating device memory. 1200 const ARM_MAIR_DEVICE_MEMORY: u8 = 0x0; 1201 1202 /// In the ARM Architecture Reference Manual, the MAIR encoding for Normal memory 1203 /// uses the format `0bxxRW` where: 1204 /// - `W` (bit 0) = Write-Allocate policy 1205 /// - `R` (bit 1) = Read-Allocate policy 1206 /// E.g., `0b0011` would allow both read and write allocation on a cache miss. 1207 /// 1208 /// ARM MAIR Write-Allocate bit (bit 0 of a cache policy nibble). 1209 const ARM_MAIR_WRITE_ALLOCATE: u8 = 0x1; 1210 /// ARM MAIR Read-Allocate bit (bit 1 of a cache policy nibble). 1211 const ARM_MAIR_READ_ALLOCATE: u8 = 0x2; 1212 1213 /// Write-back policy bit. For cacheable encodings, it is necessary but not 1214 /// sufficient to set bit 2 of the cache policy nibble. Bit 2 does not 1215 /// definitively determine write back because bit 2 is also set in `0b0100` 1216 /// which encodes Normal non-cacheable memory. 1217 const ARM_MAIR_WRITE_BACK_BIT: u8 = 0x4; 1218 1219 /// Complete cache-policy nibble encoding for Normal Non-cacheable memory. 1220 const ARM_MAIR_NON_CACHEABLE: u8 = 0x4; 1221 1222 /// Mask for the inner cache policy nibble in MAIR attribute bytes. 1223 const ARM_MAIR_INNER_MASK: u8 = 0x0f; 1224 1225 /// Check if a MAIR attribute byte represents device memory. 1226 /// 1227 /// Device memory (memory-mapped I/O, registers) cannot be cached because 1228 /// reading and writing to this memory may have side effects. 1229 fn is_device_memory(mair_attr: u8) -> bool { 1230 // In AArch64 MAIR, outer nibble only is 0 for device memory. 1231 (mair_attr >> 4) == Self::ARM_MAIR_DEVICE_MEMORY 1232 } 1233 1234 /// Check if normal memory is fully write-back cacheable. 1235 /// 1236 /// ARM MAIR has two cache policy levels (outer [7:4] and inner [3:0]). 1237 /// For memory to be truly write-back, BOTH levels must have the write-back bit set. 1238 /// If only one level is write-back, treat it as non-cacheable for GPU purposes. 1239 fn is_writeback_cacheable(mair_attr: u8) -> bool { 1240 let outer = mair_attr >> 4; 1241 let inner = mair_attr & Self::ARM_MAIR_INNER_MASK; 1242 1243 outer != Self::ARM_MAIR_NON_CACHEABLE 1244 && inner != Self::ARM_MAIR_NON_CACHEABLE 1245 && (outer & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 1246 && (inner & Self::ARM_MAIR_WRITE_BACK_BIT) != 0 1247 } 1248 1249 // Helper to encode a MEMATTR attribute from its individual fields. 1250 fn encode_attribute( 1251 alloc_w: bool, 1252 alloc_r: bool, 1253 alloc_sel: AllocPolicySelect, 1254 coherency: Coherency, 1255 memory_type: MemoryType, 1256 ) -> MMU_MEMATTR_STAGE1 { 1257 MMU_MEMATTR_STAGE1::zeroed() 1258 .with_alloc_w(alloc_w) 1259 .with_alloc_r(alloc_r) 1260 .with_alloc_sel(alloc_sel) 1261 .with_coherency(coherency) 1262 .with_memory_type(memory_type) 1263 } 1264 1265 /// Convert one MAIR attribute byte into a MEMATTR attribute. 1266 // TODO: Add a `coherent` parameter like panthor's mair_to_memattr(). 1267 // For now, assume a non-coherent system and always encode write-back 1268 // memory with MidgardInnerDomain coherency. 1269 fn attribute_from_mair(mair_attr: u8) -> MMU_MEMATTR_STAGE1 { 1270 // Device memory or non-write-back normal memory 1271 if Self::is_device_memory(mair_attr) || !Self::is_writeback_cacheable(mair_attr) { 1272 return Self::encode_attribute( 1273 false, 1274 false, 1275 AllocPolicySelect::Alloc, 1276 Coherency::MidgardInnerDomain, 1277 MemoryType::NonCacheable, 1278 ); 1279 } 1280 1281 // Write-back cacheable normal memory 1282 let inner: u8 = mair_attr & Self::ARM_MAIR_INNER_MASK; 1283 Self::encode_attribute( 1284 (inner & Self::ARM_MAIR_WRITE_ALLOCATE) != 0, 1285 (inner & Self::ARM_MAIR_READ_ALLOCATE) != 0, 1286 AllocPolicySelect::Alloc, 1287 Coherency::MidgardInnerDomain, 1288 MemoryType::WriteBack, 1289 ) 1290 } 1291 1292 /// Write one converted MAIR attribute into a corresponding MEMATTR slot. 1293 fn with_encoded_attribute(self, index: usize, attr: MMU_MEMATTR_STAGE1) -> Self { 1294 debug_assert!(index < 8); 1295 1296 let shift = index * 8; 1297 let mask = !(0xffu64 << shift); 1298 let raw = (self.into_raw() & mask) | ((u64::from(attr.into_raw())) << shift); 1299 1300 Self::from_raw(raw) 1301 } 1302 1303 /// Convert an AArch64 MAIR value into the GPU MEMATTR register encoding. 1304 /// 1305 /// Both MAIR and MEMATTR are 64-bit values with eight 8-bit memory 1306 /// attribute entries, but the bits do not map directly. The GPU MEMATTR encoding 1307 /// is less detailed than the MAIR encoding, so MAIR is converted to MEMATTR 1308 /// conservatively as follows: 1309 /// 1310 /// 1. Device memory, or Normal Memory that is not write-back cacheable, is encoded 1311 /// as GPU `NonCacheable` 1312 /// 1313 /// 2. Normal memory that is write-back cacheable is encoded as GPU `WriteBack`, 1314 /// and the inner allocation hints are preserved. 1315 pub(crate) fn from_mair(mair: u64) -> Self { 1316 mair.to_le_bytes() 1317 .into_iter() 1318 .enumerate() 1319 .fold(Self::zeroed(), |acc, (i, attr)| { 1320 acc.with_encoded_attribute(i, Self::attribute_from_mair(attr)) 1321 }) 1322 } 1323 } 1324 1325 register! { 1326 base: TyrRegisters; 1327 1328 /// Lock region address for each address space. 1329 pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { 1330 /// Lock region size. 1331 5:0 size; 1332 /// Lock region base address. 1333 63:12 base; 1334 } 1335 1336 // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two 1337 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1338 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1339 pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { 1340 31:0 value; 1341 } 1342 1343 pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { 1344 31:0 value; 1345 } 1346 } 1347 1348 /// Helpers for MMU COMMAND register. 1349 #[derive(Copy, Clone, Debug, PartialEq)] 1350 #[repr(u8)] 1351 pub(crate) enum MmuCommand { 1352 /// No operation, nothing happens. 1353 Nop = 0, 1354 /// Propagate settings to the MMU. 1355 Update = 1, 1356 /// Lock an address region. 1357 Lock = 2, 1358 /// Unlock an address region. 1359 Unlock = 3, 1360 /// Clean and invalidate the L2 cache, then unlock. 1361 FlushPt = 4, 1362 /// Clean and invalidate all caches, then unlock. 1363 FlushMem = 5, 1364 } 1365 1366 impl TryFrom<Bounded<u32, 8>> for MmuCommand { 1367 type Error = Error; 1368 1369 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1370 match val.get() { 1371 0 => Ok(MmuCommand::Nop), 1372 1 => Ok(MmuCommand::Update), 1373 2 => Ok(MmuCommand::Lock), 1374 3 => Ok(MmuCommand::Unlock), 1375 4 => Ok(MmuCommand::FlushPt), 1376 5 => Ok(MmuCommand::FlushMem), 1377 _ => Err(EINVAL), 1378 } 1379 } 1380 } 1381 1382 impl From<MmuCommand> for Bounded<u32, 8> { 1383 fn from(cmd: MmuCommand) -> Self { 1384 (cmd as u8).into() 1385 } 1386 } 1387 1388 register! { 1389 base: TyrRegisters; 1390 1391 /// MMU command register for each address space. Write only. 1392 pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 1393 7:0 command ?=> MmuCommand; 1394 } 1395 } 1396 1397 /// MMU exception types for FAULTSTATUS register. 1398 #[derive(Copy, Clone, Debug, PartialEq)] 1399 #[repr(u8)] 1400 pub(crate) enum MmuExceptionType { 1401 /// No error. 1402 Ok = 0x00, 1403 /// Invalid translation table entry, level 0. 1404 TranslationFault0 = 0xC0, 1405 /// Invalid translation table entry, level 1. 1406 TranslationFault1 = 0xC1, 1407 /// Invalid translation table entry, level 2. 1408 TranslationFault2 = 0xC2, 1409 /// Invalid translation table entry, level 3. 1410 TranslationFault3 = 0xC3, 1411 /// Invalid block descriptor. 1412 TranslationFault4 = 0xC4, 1413 /// Page permission error, level 0. 1414 PermissionFault0 = 0xC8, 1415 /// Page permission error, level 1. 1416 PermissionFault1 = 0xC9, 1417 /// Page permission error, level 2. 1418 PermissionFault2 = 0xCA, 1419 /// Page permission error, level 3. 1420 PermissionFault3 = 0xCB, 1421 /// Access flag not set, level 1. 1422 AccessFlag1 = 0xD9, 1423 /// Access flag not set, level 2. 1424 AccessFlag2 = 0xDA, 1425 /// Access flag not set, level 3. 1426 AccessFlag3 = 0xDB, 1427 /// Virtual address out of range. 1428 AddressSizeFaultIn = 0xE0, 1429 /// Physical address out of range, level 0. 1430 AddressSizeFaultOut0 = 0xE4, 1431 /// Physical address out of range, level 1. 1432 AddressSizeFaultOut1 = 0xE5, 1433 /// Physical address out of range, level 2. 1434 AddressSizeFaultOut2 = 0xE6, 1435 /// Physical address out of range, level 3. 1436 AddressSizeFaultOut3 = 0xE7, 1437 /// Page attribute error, level 0. 1438 MemoryAttributeFault0 = 0xE8, 1439 /// Page attribute error, level 1. 1440 MemoryAttributeFault1 = 0xE9, 1441 /// Page attribute error, level 2. 1442 MemoryAttributeFault2 = 0xEA, 1443 /// Page attribute error, level 3. 1444 MemoryAttributeFault3 = 0xEB, 1445 } 1446 1447 impl TryFrom<Bounded<u32, 8>> for MmuExceptionType { 1448 type Error = Error; 1449 1450 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1451 match val.get() { 1452 0x00 => Ok(MmuExceptionType::Ok), 1453 0xC0 => Ok(MmuExceptionType::TranslationFault0), 1454 0xC1 => Ok(MmuExceptionType::TranslationFault1), 1455 0xC2 => Ok(MmuExceptionType::TranslationFault2), 1456 0xC3 => Ok(MmuExceptionType::TranslationFault3), 1457 0xC4 => Ok(MmuExceptionType::TranslationFault4), 1458 0xC8 => Ok(MmuExceptionType::PermissionFault0), 1459 0xC9 => Ok(MmuExceptionType::PermissionFault1), 1460 0xCA => Ok(MmuExceptionType::PermissionFault2), 1461 0xCB => Ok(MmuExceptionType::PermissionFault3), 1462 0xD9 => Ok(MmuExceptionType::AccessFlag1), 1463 0xDA => Ok(MmuExceptionType::AccessFlag2), 1464 0xDB => Ok(MmuExceptionType::AccessFlag3), 1465 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), 1466 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), 1467 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), 1468 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), 1469 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), 1470 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), 1471 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), 1472 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), 1473 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), 1474 _ => Err(EINVAL), 1475 } 1476 } 1477 } 1478 1479 impl From<MmuExceptionType> for Bounded<u32, 8> { 1480 fn from(exc: MmuExceptionType) -> Self { 1481 (exc as u8).into() 1482 } 1483 } 1484 1485 /// Access type for MMU faults. 1486 #[derive(Copy, Clone, Debug, PartialEq)] 1487 #[repr(u8)] 1488 pub(crate) enum MmuAccessType { 1489 /// An atomic (read/write) transaction. 1490 Atomic = 0, 1491 /// An execute transaction. 1492 Execute = 1, 1493 /// A read transaction. 1494 Read = 2, 1495 /// A write transaction. 1496 Write = 3, 1497 } 1498 1499 impl From<Bounded<u32, 2>> for MmuAccessType { 1500 fn from(val: Bounded<u32, 2>) -> Self { 1501 match val.get() { 1502 0 => MmuAccessType::Atomic, 1503 1 => MmuAccessType::Execute, 1504 2 => MmuAccessType::Read, 1505 3 => MmuAccessType::Write, 1506 _ => unreachable!(), 1507 } 1508 } 1509 } 1510 1511 impl From<MmuAccessType> for Bounded<u32, 2> { 1512 fn from(access: MmuAccessType) -> Self { 1513 Bounded::try_new(access as u32).unwrap() 1514 } 1515 } 1516 1517 register! { 1518 base: TyrRegisters; 1519 1520 /// Fault status register for each address space. Read only. 1521 pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { 1522 /// Exception type. 1523 7:0 exception_type ?=> MmuExceptionType; 1524 /// Access type. 1525 9:8 access_type => MmuAccessType; 1526 /// ID of the source that triggered the fault. 1527 31:16 source_id; 1528 } 1529 1530 /// Fault address for each address space. Read only. 1531 pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { 1532 31:0 pointer; 1533 } 1534 1535 pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { 1536 31:0 pointer; 1537 } 1538 1539 /// MMU status register for each address space. Read only. 1540 pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { 1541 /// External address space command is active, a 1-bit boolean flag. 1542 0:0 active_ext => bool; 1543 /// Internal address space command is active, a 1-bit boolean flag. 1544 1:1 active_int => bool; 1545 } 1546 } 1547 1548 /// Helpers for TRANSCFG register. 1549 /// 1550 /// Address space mode for TRANSCFG register. 1551 #[derive(Copy, Clone, Debug, PartialEq)] 1552 #[repr(u8)] 1553 pub(crate) enum AddressSpaceMode { 1554 /// The MMU forces all memory access to fail with a decode fault. 1555 Unmapped = 1, 1556 /// All input addresses map to the same output address (deprecated). 1557 Identity = 2, 1558 /// Translation tables interpreted according to AArch64 4kB granule specification. 1559 Aarch64_4K = 6, 1560 /// Translation tables interpreted according to AArch64 64kB granule specification. 1561 Aarch64_64K = 8, 1562 } 1563 1564 impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode { 1565 type Error = Error; 1566 1567 fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> { 1568 match val.get() { 1569 1 => Ok(AddressSpaceMode::Unmapped), 1570 2 => Ok(AddressSpaceMode::Identity), 1571 6 => Ok(AddressSpaceMode::Aarch64_4K), 1572 8 => Ok(AddressSpaceMode::Aarch64_64K), 1573 _ => Err(EINVAL), 1574 } 1575 } 1576 } 1577 1578 impl From<AddressSpaceMode> for Bounded<u64, 4> { 1579 fn from(mode: AddressSpaceMode) -> Self { 1580 Bounded::try_new(mode as u64).unwrap() 1581 } 1582 } 1583 1584 /// Input address range restriction for TRANSCFG register. 1585 #[derive(Copy, Clone, Debug, PartialEq)] 1586 #[repr(u8)] 1587 pub(crate) enum InaBits { 1588 /// Invalid VA range (reset value). 1589 Reset = 0, 1590 /// 48-bit VA range. 1591 Bits48 = 7, 1592 /// 47-bit VA range. 1593 Bits47 = 8, 1594 /// 46-bit VA range. 1595 Bits46 = 9, 1596 /// 45-bit VA range. 1597 Bits45 = 10, 1598 /// 44-bit VA range. 1599 Bits44 = 11, 1600 /// 43-bit VA range. 1601 Bits43 = 12, 1602 /// 42-bit VA range. 1603 Bits42 = 13, 1604 /// 41-bit VA range. 1605 Bits41 = 14, 1606 /// 40-bit VA range. 1607 Bits40 = 15, 1608 /// 39-bit VA range. 1609 Bits39 = 16, 1610 /// 38-bit VA range. 1611 Bits38 = 17, 1612 /// 37-bit VA range. 1613 Bits37 = 18, 1614 /// 36-bit VA range. 1615 Bits36 = 19, 1616 /// 35-bit VA range. 1617 Bits35 = 20, 1618 /// 34-bit VA range. 1619 Bits34 = 21, 1620 /// 33-bit VA range. 1621 Bits33 = 22, 1622 /// 32-bit VA range. 1623 Bits32 = 23, 1624 /// 31-bit VA range. 1625 Bits31 = 24, 1626 /// 30-bit VA range. 1627 Bits30 = 25, 1628 /// 29-bit VA range. 1629 Bits29 = 26, 1630 /// 28-bit VA range. 1631 Bits28 = 27, 1632 /// 27-bit VA range. 1633 Bits27 = 28, 1634 /// 26-bit VA range. 1635 Bits26 = 29, 1636 /// 25-bit VA range. 1637 Bits25 = 30, 1638 } 1639 1640 impl TryFrom<Bounded<u64, 5>> for InaBits { 1641 type Error = Error; 1642 1643 fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> { 1644 match val.get() { 1645 0 => Ok(InaBits::Reset), 1646 7 => Ok(InaBits::Bits48), 1647 8 => Ok(InaBits::Bits47), 1648 9 => Ok(InaBits::Bits46), 1649 10 => Ok(InaBits::Bits45), 1650 11 => Ok(InaBits::Bits44), 1651 12 => Ok(InaBits::Bits43), 1652 13 => Ok(InaBits::Bits42), 1653 14 => Ok(InaBits::Bits41), 1654 15 => Ok(InaBits::Bits40), 1655 16 => Ok(InaBits::Bits39), 1656 17 => Ok(InaBits::Bits38), 1657 18 => Ok(InaBits::Bits37), 1658 19 => Ok(InaBits::Bits36), 1659 20 => Ok(InaBits::Bits35), 1660 21 => Ok(InaBits::Bits34), 1661 22 => Ok(InaBits::Bits33), 1662 23 => Ok(InaBits::Bits32), 1663 24 => Ok(InaBits::Bits31), 1664 25 => Ok(InaBits::Bits30), 1665 26 => Ok(InaBits::Bits29), 1666 27 => Ok(InaBits::Bits28), 1667 28 => Ok(InaBits::Bits27), 1668 29 => Ok(InaBits::Bits26), 1669 30 => Ok(InaBits::Bits25), 1670 _ => Err(EINVAL), 1671 } 1672 } 1673 } 1674 1675 impl From<InaBits> for Bounded<u64, 5> { 1676 fn from(bits: InaBits) -> Self { 1677 Bounded::try_new(bits as u64).unwrap() 1678 } 1679 } 1680 1681 /// Translation table memory attributes for TRANSCFG register. 1682 #[derive(Copy, Clone, Debug, PartialEq)] 1683 #[repr(u8)] 1684 pub(crate) enum PtwMemattr { 1685 /// Invalid (reset value, not valid for enabled address space). 1686 Invalid = 0, 1687 /// Normal memory, inner/outer non-cacheable. 1688 NonCacheable = 1, 1689 /// Normal memory, inner/outer write-back cacheable. 1690 WriteBack = 2, 1691 } 1692 1693 impl TryFrom<Bounded<u64, 2>> for PtwMemattr { 1694 type Error = Error; 1695 1696 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1697 match val.get() { 1698 0 => Ok(PtwMemattr::Invalid), 1699 1 => Ok(PtwMemattr::NonCacheable), 1700 2 => Ok(PtwMemattr::WriteBack), 1701 _ => Err(EINVAL), 1702 } 1703 } 1704 } 1705 1706 impl From<PtwMemattr> for Bounded<u64, 2> { 1707 fn from(attr: PtwMemattr) -> Self { 1708 Bounded::try_new(attr as u64).unwrap() 1709 } 1710 } 1711 1712 /// Translation table memory shareability for TRANSCFG register. 1713 #[derive(Copy, Clone, Debug, PartialEq)] 1714 #[repr(u8)] 1715 #[allow(clippy::enum_variant_names)] 1716 pub(crate) enum PtwShareability { 1717 /// Non-shareable. 1718 NonShareable = 0, 1719 /// Outer shareable. 1720 OuterShareable = 2, 1721 /// Inner shareable. 1722 InnerShareable = 3, 1723 } 1724 1725 impl TryFrom<Bounded<u64, 2>> for PtwShareability { 1726 type Error = Error; 1727 1728 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1729 match val.get() { 1730 0 => Ok(PtwShareability::NonShareable), 1731 2 => Ok(PtwShareability::OuterShareable), 1732 3 => Ok(PtwShareability::InnerShareable), 1733 _ => Err(EINVAL), 1734 } 1735 } 1736 } 1737 1738 impl From<PtwShareability> for Bounded<u64, 2> { 1739 fn from(sh: PtwShareability) -> Self { 1740 Bounded::try_new(sh as u64).unwrap() 1741 } 1742 } 1743 1744 register! { 1745 base: TyrRegisters; 1746 1747 /// Translation configuration and control. 1748 pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { 1749 /// Address space mode. 1750 3:0 mode ?=> AddressSpaceMode; 1751 /// Address input restriction. 1752 10:6 ina_bits ?=> InaBits; 1753 /// Address output restriction. 1754 18:14 outa_bits; 1755 /// Translation table concatenation enable, a 1-bit boolean flag. 1756 22:22 sl_concat_en => bool; 1757 /// Translation table memory attributes. 1758 25:24 ptw_memattr ?=> PtwMemattr; 1759 /// Translation table memory shareability. 1760 29:28 ptw_sh ?=> PtwShareability; 1761 /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. 1762 30:30 r_allocate => bool; 1763 /// Disable hierarchical access permissions. 1764 33:33 disable_hier_ap => bool; 1765 /// Disable access fault checking. 1766 34:34 disable_af_fault => bool; 1767 /// Disable execution on all writable pages. 1768 35:35 wxn => bool; 1769 /// Enable execution on readable pages. 1770 36:36 xreadable => bool; 1771 /// Page-based hardware attributes for translation table walks. 1772 63:60 ptw_pbha; 1773 } 1774 1775 // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two 1776 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1777 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1778 pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { 1779 31:0 value; 1780 } 1781 1782 pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { 1783 31:0 value; 1784 } 1785 1786 /// Extra fault information for each address space. Read only. 1787 pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { 1788 31:0 value; 1789 } 1790 1791 pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { 1792 31:0 value; 1793 } 1794 } 1795 } 1796 } 1797 1798 /// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. 1799 pub(crate) mod doorbell_block { 1800 use kernel::register; 1801 1802 use crate::driver::TyrRegisters; 1803 1804 /// Number of doorbells available. 1805 pub(crate) const NUM_DOORBELLS: usize = 64; 1806 1807 /// Doorbell block stride (64KiB). 1808 /// 1809 /// Each block occupies a full page, allowing it to be mapped 1810 /// separately into a virtual address space. 1811 const STRIDE: usize = 0x10000; 1812 1813 register! { 1814 base: TyrRegisters; 1815 1816 /// Doorbell request register. Write-only. 1817 pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { 1818 /// Doorbell set. Writing 1 triggers the doorbell. 1819 0:0 ring => bool; 1820 } 1821 } 1822 } 1823