1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 3 //! # Definitions 4 //! 5 //! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) 6 //! from the command stream. 7 //! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular 8 //! job or sequence of jobs. The instructions exist in one or more command buffers. 9 //! - **CSF**: Command Stream Frontend - The interface and implementation for job submission 10 //! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS 11 //! interfaces. 12 //! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple 13 //! CSGs, and each CSG contains multiple CSs. 14 //! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the 15 //! iterator control aspects. Implements the CSF in conjunction with the MCU. 16 //! - **GLB**: Global - Prefix for global interface registers that control operations common to 17 //! all CSs. 18 //! - **JASID**: Job Address Space ID - Identifies the address space for a job. 19 //! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream 20 //! hardware. 21 //! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. 22 23 // We don't expect that all the registers and fields will be used, even in the 24 // future. 25 // 26 // Nevertheless, it is useful to have most of them defined, like the C driver 27 // does. 28 #![allow(dead_code)] 29 30 /// Combine two 32-bit values into a single 64-bit value. 31 pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { 32 (u64::from(lo)) | ((u64::from(hi)) << 32) 33 } 34 35 /// Read a logical 64-bit value from split 32-bit registers without tearing. 36 pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { 37 loop { 38 let hi1 = hi_read(); 39 let lo = lo_read(); 40 let hi2 = hi_read(); 41 42 if hi1 == hi2 { 43 return join_u64(lo, hi1); 44 } 45 } 46 } 47 48 /// These registers correspond to the GPU_CONTROL register page. 49 /// They are involved in GPU configuration and control. 50 pub(crate) mod gpu_control { 51 use kernel::{ 52 num::Bounded, 53 prelude::*, 54 register, 55 uapi, // 56 }; 57 58 register! { 59 /// GPU identification register. 60 pub(crate) GPU_ID(u32) @ 0x0 { 61 /// Status of the GPU release. 62 3:0 ver_status; 63 /// Minor release version number. 64 11:4 ver_minor; 65 /// Major release version number. 66 15:12 ver_major; 67 /// Product identifier. 68 19:16 prod_major; 69 /// Architecture patch revision. 70 23:20 arch_rev; 71 /// Architecture minor revision. 72 27:24 arch_minor; 73 /// Architecture major revision. 74 31:28 arch_major; 75 } 76 77 /// Level 2 cache features register. 78 pub(crate) L2_FEATURES(u32) @ 0x4 { 79 /// Cache line size. 80 7:0 line_size; 81 /// Cache associativity. 82 15:8 associativity; 83 /// Cache slice size. 84 23:16 cache_size; 85 /// External bus width. 86 31:24 bus_width; 87 } 88 89 /// Shader core features. 90 pub(crate) CORE_FEATURES(u32) @ 0x8 { 91 /// Shader core variant. 92 7:0 core_variant; 93 } 94 95 /// Tiler features. 96 pub(crate) TILER_FEATURES(u32) @ 0xc { 97 /// Log of the tiler's bin size. 98 5:0 bin_size; 99 /// Maximum number of active levels. 100 11:8 max_levels; 101 } 102 103 /// Memory system features. 104 pub(crate) MEM_FEATURES(u32) @ 0x10 { 105 0:0 coherent_core_group => bool; 106 1:1 coherent_super_group => bool; 107 11:8 l2_slices; 108 } 109 110 /// Memory management unit features. 111 pub(crate) MMU_FEATURES(u32) @ 0x14 { 112 /// Number of bits supported in virtual addresses. 113 7:0 va_bits; 114 /// Number of bits supported in physical addresses. 115 15:8 pa_bits; 116 } 117 118 /// Address spaces present. 119 pub(crate) AS_PRESENT(u32) @ 0x18 { 120 31:0 present; 121 } 122 123 /// CSF version information. 124 pub(crate) CSF_ID(u32) @ 0x1c { 125 /// MCU revision ID. 126 3:0 mcu_rev; 127 /// MCU minor revision number. 128 9:4 mcu_minor; 129 /// MCU major revision number. 130 15:10 mcu_major; 131 /// CSHW revision ID. 132 19:16 cshw_rev; 133 /// CSHW minor revision number. 134 25:20 cshw_minor; 135 /// CSHW major revision number. 136 31:26 cshw_major; 137 } 138 139 /// IRQ sources raw status. 140 /// Writing to this register forces bits on, but does not clear them. 141 pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { 142 /// A GPU fault has occurred, a 1-bit boolean flag. 143 0:0 gpu_fault => bool; 144 /// A GPU fault has occurred, a 1-bit boolean flag. 145 1:1 gpu_protected_fault => bool; 146 /// Reset has completed, a 1-bit boolean flag. 147 8:8 reset_completed => bool; 148 /// Set when a single power domain has powered up or down, a 1-bit boolean flag. 149 9:9 power_changed_single => bool; 150 /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. 151 10:10 power_changed_all => bool; 152 /// Set when cache cleaning has completed, a 1-bit boolean flag. 153 17:17 clean_caches_completed => bool; 154 /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. 155 18:18 doorbell_mirror => bool; 156 /// MCU requires attention, a 1-bit boolean flag. 157 19:19 mcu_status => bool; 158 } 159 160 /// IRQ sources to clear. Write only. 161 pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { 162 /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. 163 0:0 gpu_fault => bool; 164 /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 165 1:1 gpu_protected_fault => bool; 166 /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. 167 8:8 reset_completed => bool; 168 /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 169 9:9 power_changed_single => bool; 170 /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 171 10:10 power_changed_all => bool; 172 /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 173 17:17 clean_caches_completed => bool; 174 /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. 175 19:19 mcu_status => bool; 176 } 177 178 /// IRQ sources enabled. 179 pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { 180 /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. 181 0:0 gpu_fault => bool; 182 /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 183 1:1 gpu_protected_fault => bool; 184 /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. 185 8:8 reset_completed => bool; 186 /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 187 9:9 power_changed_single => bool; 188 /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 189 10:10 power_changed_all => bool; 190 /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 191 17:17 clean_caches_completed => bool; 192 /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. 193 18:18 doorbell_mirror => bool; 194 /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. 195 19:19 mcu_status => bool; 196 } 197 198 /// IRQ status for enabled sources. Read only. 199 pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { 200 /// GPU_FAULT interrupt status, a 1-bit boolean flag. 201 0:0 gpu_fault => bool; 202 /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. 203 1:1 gpu_protected_fault => bool; 204 /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. 205 8:8 reset_completed => bool; 206 /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. 207 9:9 power_changed_single => bool; 208 /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. 209 10:10 power_changed_all => bool; 210 /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. 211 17:17 clean_caches_completed => bool; 212 /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. 213 18:18 doorbell_mirror => bool; 214 /// MCU_STATUS interrupt status, a 1-bit boolean flag. 215 19:19 mcu_status => bool; 216 } 217 } 218 219 /// Helpers for GPU_COMMAND Register 220 #[derive(Copy, Clone, Debug, PartialEq)] 221 #[repr(u8)] 222 pub(crate) enum GpuCommand { 223 /// No operation. This is the default value. 224 Nop = 0, 225 /// Reset the GPU. 226 Reset = 1, 227 /// Flush caches. 228 FlushCaches = 4, 229 /// Clear GPU faults. 230 ClearFault = 7, 231 } 232 233 impl TryFrom<Bounded<u32, 8>> for GpuCommand { 234 type Error = Error; 235 236 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 237 match val.get() { 238 0 => Ok(GpuCommand::Nop), 239 1 => Ok(GpuCommand::Reset), 240 4 => Ok(GpuCommand::FlushCaches), 241 7 => Ok(GpuCommand::ClearFault), 242 _ => Err(EINVAL), 243 } 244 } 245 } 246 247 impl From<GpuCommand> for Bounded<u32, 8> { 248 fn from(cmd: GpuCommand) -> Self { 249 (cmd as u8).into() 250 } 251 } 252 253 /// Reset mode for [`GPU_COMMAND::reset()`]. 254 #[derive(Copy, Clone, Debug, PartialEq)] 255 #[repr(u8)] 256 pub(crate) enum ResetMode { 257 /// Stop all external bus interfaces, then reset the entire GPU. 258 SoftReset = 1, 259 /// Force a full GPU reset. 260 HardReset = 2, 261 } 262 263 impl TryFrom<Bounded<u32, 4>> for ResetMode { 264 type Error = Error; 265 266 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 267 match val.get() { 268 1 => Ok(ResetMode::SoftReset), 269 2 => Ok(ResetMode::HardReset), 270 _ => Err(EINVAL), 271 } 272 } 273 } 274 275 impl From<ResetMode> for Bounded<u32, 4> { 276 fn from(mode: ResetMode) -> Self { 277 Bounded::try_new(mode as u32).unwrap() 278 } 279 } 280 281 /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. 282 #[derive(Copy, Clone, Debug, PartialEq)] 283 #[repr(u8)] 284 pub(crate) enum FlushMode { 285 /// No flush. 286 None = 0, 287 /// Clean the caches. 288 Clean = 1, 289 /// Invalidate the caches. 290 Invalidate = 2, 291 /// Clean and invalidate the caches. 292 CleanInvalidate = 3, 293 } 294 295 impl TryFrom<Bounded<u32, 4>> for FlushMode { 296 type Error = Error; 297 298 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 299 match val.get() { 300 0 => Ok(FlushMode::None), 301 1 => Ok(FlushMode::Clean), 302 2 => Ok(FlushMode::Invalidate), 303 3 => Ok(FlushMode::CleanInvalidate), 304 _ => Err(EINVAL), 305 } 306 } 307 } 308 309 impl From<FlushMode> for Bounded<u32, 4> { 310 fn from(mode: FlushMode) -> Self { 311 Bounded::try_new(mode as u32).unwrap() 312 } 313 } 314 315 register! { 316 /// GPU command register. 317 /// 318 /// Use the constructor methods to create commands: 319 /// - [`GPU_COMMAND::nop()`] 320 /// - [`GPU_COMMAND::reset()`] 321 /// - [`GPU_COMMAND::flush_caches()`] 322 /// - [`GPU_COMMAND::clear_fault()`] 323 pub(crate) GPU_COMMAND (u32) @ 0x30 { 324 7:0 command ?=> GpuCommand; 325 } 326 /// Internal alias for GPU_COMMAND in reset mode. 327 /// Use [`GPU_COMMAND::reset()`] instead. 328 GPU_COMMAND_RESET (u32) => GPU_COMMAND { 329 7:0 command ?=> GpuCommand; 330 11:8 reset_mode ?=> ResetMode; 331 } 332 333 /// Internal alias for GPU_COMMAND in cache flush mode. 334 /// Use [`GPU_COMMAND::flush_caches()`] instead. 335 GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { 336 7:0 command ?=> GpuCommand; 337 /// L2 cache flush mode. 338 11:8 l2_flush ?=> FlushMode; 339 /// Shader core load/store cache flush mode. 340 15:12 lsc_flush ?=> FlushMode; 341 /// Shader core other caches flush mode. 342 19:16 other_flush ?=> FlushMode; 343 } 344 } 345 346 impl GPU_COMMAND { 347 /// Create a NOP command. 348 pub(crate) fn nop() -> Self { 349 Self::zeroed() 350 } 351 352 /// Create a reset command with the specified reset mode. 353 pub(crate) fn reset(mode: ResetMode) -> Self { 354 Self::from_raw( 355 GPU_COMMAND_RESET::zeroed() 356 .with_command(GpuCommand::Reset) 357 .with_reset_mode(mode) 358 .into_raw(), 359 ) 360 } 361 362 /// Create a cache flush command with the specified flush modes. 363 pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { 364 Self::from_raw( 365 GPU_COMMAND_FLUSH::zeroed() 366 .with_command(GpuCommand::FlushCaches) 367 .with_l2_flush(l2) 368 .with_lsc_flush(lsc) 369 .with_other_flush(other) 370 .into_raw(), 371 ) 372 } 373 374 /// Create a clear fault command. 375 pub(crate) fn clear_fault() -> Self { 376 Self::zeroed().with_command(GpuCommand::ClearFault) 377 } 378 } 379 380 register! { 381 /// GPU status register. Read only. 382 pub(crate) GPU_STATUS(u32) @ 0x34 { 383 /// GPU active, a 1-bit boolean flag. 384 0:0 gpu_active => bool; 385 /// Power manager active, a 1-bit boolean flag 386 1:1 pwr_active => bool; 387 /// Page fault active, a 1-bit boolean flag. 388 4:4 page_fault => bool; 389 /// Protected mode active, a 1-bit boolean flag. 390 7:7 protected_mode_active => bool; 391 /// Debug mode active, a 1-bit boolean flag. 392 8:8 gpu_dbg_enabled => bool; 393 } 394 } 395 396 #[derive(Copy, Clone, Debug, PartialEq)] 397 #[repr(u8)] 398 pub(crate) enum ExceptionType { 399 /// Exception type: No error. 400 Ok = 0x00, 401 /// Exception type: GPU external bus error. 402 GpuBusFault = 0x80, 403 /// Exception type: GPU shareability error. 404 GpuShareabilityFault = 0x88, 405 /// Exception type: System shareability error. 406 SystemShareabilityFault = 0x89, 407 /// Exception type: GPU cacheability error. 408 GpuCacheabilityFault = 0x8A, 409 } 410 411 impl TryFrom<Bounded<u32, 8>> for ExceptionType { 412 type Error = Error; 413 414 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 415 match val.get() { 416 0x00 => Ok(ExceptionType::Ok), 417 0x80 => Ok(ExceptionType::GpuBusFault), 418 0x88 => Ok(ExceptionType::GpuShareabilityFault), 419 0x89 => Ok(ExceptionType::SystemShareabilityFault), 420 0x8A => Ok(ExceptionType::GpuCacheabilityFault), 421 _ => Err(EINVAL), 422 } 423 } 424 } 425 426 impl From<ExceptionType> for Bounded<u32, 8> { 427 fn from(exc: ExceptionType) -> Self { 428 (exc as u8).into() 429 } 430 } 431 432 #[derive(Copy, Clone, Debug, PartialEq)] 433 #[repr(u8)] 434 pub(crate) enum AccessType { 435 /// Access type: An atomic (read/write) transaction. 436 Atomic = 0, 437 /// Access type: An execute transaction. 438 Execute = 1, 439 /// Access type: A read transaction. 440 Read = 2, 441 /// Access type: A write transaction. 442 Write = 3, 443 } 444 445 impl From<Bounded<u32, 2>> for AccessType { 446 fn from(val: Bounded<u32, 2>) -> Self { 447 match val.get() { 448 0 => AccessType::Atomic, 449 1 => AccessType::Execute, 450 2 => AccessType::Read, 451 3 => AccessType::Write, 452 _ => unreachable!(), 453 } 454 } 455 } 456 457 impl From<AccessType> for Bounded<u32, 2> { 458 fn from(access: AccessType) -> Self { 459 Bounded::try_new(access as u32).unwrap() 460 } 461 } 462 463 register! { 464 /// GPU fault status register. Read only. 465 pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { 466 /// Exception type. 467 7:0 exception_type ?=> ExceptionType; 468 /// Access type. 469 9:8 access_type => AccessType; 470 /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. 471 10:10 address_valid => bool; 472 /// The JASID field is valid, a 1-bit boolean flag. 473 11:11 jasid_valid => bool; 474 /// JASID of the fault, if known. 475 15:12 jasid; 476 /// ID of the source that triggered the fault. 477 31:16 source_id; 478 } 479 480 /// GPU fault address. Read only. 481 /// Once a fault is reported, it must be manually cleared by issuing a 482 /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU 483 /// faults will be reported until the previous fault has been cleared. 484 pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { 485 31:0 pointer; 486 } 487 488 pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { 489 31:0 pointer; 490 } 491 492 /// Level 2 cache configuration. 493 pub(crate) L2_CONFIG(u32) @ 0x48 { 494 /// Requested cache size. 495 23:16 cache_size; 496 /// Requested hash function index. 497 31:24 hash_function; 498 } 499 500 /// Global time stamp offset. 501 pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { 502 31:0 offset; 503 } 504 505 pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { 506 31:0 offset; 507 } 508 509 /// GPU cycle counter. Read only. 510 pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { 511 31:0 count; 512 } 513 514 pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { 515 31:0 count; 516 } 517 518 /// Global time stamp. Read only. 519 pub(crate) TIMESTAMP_LO(u32) @ 0x98 { 520 31:0 timestamp; 521 } 522 523 pub(crate) TIMESTAMP_HI(u32) @ 0x9c { 524 31:0 timestamp; 525 } 526 527 /// Maximum number of threads per core. Read only constant. 528 pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { 529 31:0 threads; 530 } 531 532 /// Maximum number of threads per workgroup. Read only constant. 533 pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { 534 31:0 threads; 535 } 536 537 /// Maximum number of threads per barrier. Read only constant. 538 pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { 539 31:0 threads; 540 } 541 542 /// Thread features. Read only constant. 543 pub(crate) THREAD_FEATURES(u32) @ 0xac { 544 /// Total number of registers per core. 545 21:0 max_registers; 546 /// Implementation technology type. 547 23:22 implementation_technology; 548 /// Maximum number of compute tasks waiting. 549 31:24 max_task_queue; 550 } 551 552 /// Support flags for compressed texture formats. Read only constant. 553 /// 554 /// A bitmap where each bit indicates support for a specific compressed texture format. 555 /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). 556 pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { 557 31:0 supported_formats; 558 } 559 560 /// Shader core present bitmap. Read only constant. 561 pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { 562 31:0 value; 563 } 564 565 pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { 566 31:0 value; 567 } 568 569 /// Tiler present bitmap. Read only constant. 570 pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { 571 31:0 present; 572 } 573 574 pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { 575 31:0 present; 576 } 577 578 /// L2 cache present bitmap. Read only constant. 579 pub(crate) L2_PRESENT_LO(u32) @ 0x120 { 580 31:0 present; 581 } 582 583 pub(crate) L2_PRESENT_HI(u32) @ 0x124 { 584 31:0 present; 585 } 586 587 /// Shader core ready bitmap. Read only. 588 pub(crate) SHADER_READY_LO(u32) @ 0x140 { 589 31:0 ready; 590 } 591 592 pub(crate) SHADER_READY_HI(u32) @ 0x144 { 593 31:0 ready; 594 } 595 596 /// Tiler ready bitmap. Read only. 597 pub(crate) TILER_READY_LO(u32) @ 0x150 { 598 31:0 ready; 599 } 600 601 pub(crate) TILER_READY_HI(u32) @ 0x154 { 602 31:0 ready; 603 } 604 605 /// L2 ready bitmap. Read only. 606 pub(crate) L2_READY_LO(u32) @ 0x160 { 607 31:0 ready; 608 } 609 610 pub(crate) L2_READY_HI(u32) @ 0x164 { 611 31:0 ready; 612 } 613 614 /// Shader core power up bitmap. 615 pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { 616 31:0 request; 617 } 618 619 pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { 620 31:0 request; 621 } 622 623 /// Tiler power up bitmap. 624 pub(crate) TILER_PWRON_LO(u32) @ 0x190 { 625 31:0 request; 626 } 627 628 pub(crate) TILER_PWRON_HI(u32) @ 0x194 { 629 31:0 request; 630 } 631 632 /// L2 power up bitmap. 633 pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { 634 31:0 request; 635 } 636 637 pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { 638 31:0 request; 639 } 640 641 /// Shader core power down bitmap. 642 pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { 643 31:0 request; 644 } 645 646 pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { 647 31:0 request; 648 } 649 650 /// Tiler power down bitmap. 651 pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { 652 31:0 request; 653 } 654 655 pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { 656 31:0 request; 657 } 658 659 /// L2 power down bitmap. 660 pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { 661 31:0 request; 662 } 663 664 pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { 665 31:0 request; 666 } 667 668 /// Shader core power transition bitmap. Read-only. 669 pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { 670 31:0 changing; 671 } 672 673 pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { 674 31:0 changing; 675 } 676 677 /// Tiler power transition bitmap. Read-only. 678 pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { 679 31:0 changing; 680 } 681 682 pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { 683 31:0 changing; 684 } 685 686 /// L2 power transition bitmap. Read-only. 687 pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { 688 31:0 changing; 689 } 690 691 pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { 692 31:0 changing; 693 } 694 695 /// Shader core active bitmap. Read-only. 696 pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { 697 31:0 active; 698 } 699 700 pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { 701 31:0 active; 702 } 703 704 /// Tiler active bitmap. Read-only. 705 pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { 706 31:0 active; 707 } 708 709 pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { 710 31:0 active; 711 } 712 713 /// L2 active bitmap. Read-only. 714 pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { 715 31:0 active; 716 } 717 718 pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { 719 31:0 active; 720 } 721 722 /// Revision ID. Read only constant. 723 pub(crate) REVIDR(u32) @ 0x280 { 724 31:0 revision; 725 } 726 727 /// Coherency features present. Read only constant. 728 /// Supported protocols on the interconnect between the GPU and the 729 /// system into which it is integrated. 730 pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { 731 /// ACE-Lite protocol supported, a 1-bit boolean flag. 732 0:0 ace_lite => bool; 733 /// ACE protocol supported, a 1-bit boolean flag. 734 1:1 ace => bool; 735 } 736 } 737 738 #[derive(Copy, Clone, Debug, PartialEq)] 739 #[repr(u8)] 740 pub(crate) enum CoherencyMode { 741 /// ACE-Lite coherency protocol. 742 AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, 743 /// ACE coherency protocol. 744 Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, 745 /// No coherency protocol. 746 None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, 747 } 748 749 impl TryFrom<Bounded<u32, 32>> for CoherencyMode { 750 type Error = Error; 751 752 fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> { 753 match val.get() { 754 0 => Ok(CoherencyMode::AceLite), 755 1 => Ok(CoherencyMode::Ace), 756 31 => Ok(CoherencyMode::None), 757 _ => Err(EINVAL), 758 } 759 } 760 } 761 762 impl From<CoherencyMode> for Bounded<u32, 32> { 763 fn from(mode: CoherencyMode) -> Self { 764 (mode as u8).into() 765 } 766 } 767 768 register! { 769 /// Coherency enable. An index of which coherency protocols should be used. 770 /// This register only selects the protocol for coherency messages on the 771 /// interconnect. This is not to enable or disable coherency controlled by MMU. 772 pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { 773 31:0 l2_cache_protocol_select ?=> CoherencyMode; 774 } 775 } 776 777 /// Helpers for MCU_CONTROL register 778 #[derive(Copy, Clone, Debug, PartialEq)] 779 #[repr(u8)] 780 pub(crate) enum McuControlMode { 781 /// Disable the MCU. 782 Disable = 0, 783 /// Enable the MCU. 784 Enable = 1, 785 /// Enable the MCU to execute and automatically reboot after a fast reset. 786 Auto = 2, 787 } 788 789 impl TryFrom<Bounded<u32, 2>> for McuControlMode { 790 type Error = Error; 791 792 fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> { 793 match val.get() { 794 0 => Ok(McuControlMode::Disable), 795 1 => Ok(McuControlMode::Enable), 796 2 => Ok(McuControlMode::Auto), 797 _ => Err(EINVAL), 798 } 799 } 800 } 801 802 impl From<McuControlMode> for Bounded<u32, 2> { 803 fn from(mode: McuControlMode) -> Self { 804 Bounded::try_new(mode as u32).unwrap() 805 } 806 } 807 808 register! { 809 /// MCU control. 810 pub(crate) MCU_CONTROL(u32) @ 0x700 { 811 /// Request MCU state change. 812 1:0 req ?=> McuControlMode; 813 } 814 } 815 816 /// Helpers for MCU_STATUS register 817 #[derive(Copy, Clone, Debug, PartialEq)] 818 #[repr(u8)] 819 pub(crate) enum McuStatus { 820 /// MCU is disabled. 821 Disabled = 0, 822 /// MCU is enabled. 823 Enabled = 1, 824 /// The MCU has halted by itself in an orderly manner to enable the core group to be 825 /// powered down. 826 Halt = 2, 827 /// The MCU has encountered an error that prevents it from continuing. 828 Fatal = 3, 829 } 830 831 impl From<Bounded<u32, 2>> for McuStatus { 832 fn from(val: Bounded<u32, 2>) -> Self { 833 match val.get() { 834 0 => McuStatus::Disabled, 835 1 => McuStatus::Enabled, 836 2 => McuStatus::Halt, 837 3 => McuStatus::Fatal, 838 _ => unreachable!(), 839 } 840 } 841 } 842 843 impl From<McuStatus> for Bounded<u32, 2> { 844 fn from(status: McuStatus) -> Self { 845 Bounded::try_new(status as u32).unwrap() 846 } 847 } 848 849 register! { 850 /// MCU status. Read only. 851 pub(crate) MCU_STATUS(u32) @ 0x704 { 852 /// Read current state of MCU. 853 1:0 value => McuStatus; 854 } 855 } 856 } 857 858 /// These registers correspond to the JOB_CONTROL register page. 859 /// They are involved in communication between the firmware running on the MCU and the host. 860 pub(crate) mod job_control { 861 use kernel::register; 862 863 register! { 864 /// Raw status of job interrupts. 865 /// 866 /// Write to this register to trigger these interrupts. 867 /// Writing a 1 to a bit forces that bit on. 868 pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { 869 /// CSG request. These bits indicate that CSGn requires attention from the host. 870 30:0 csg; 871 /// GLB request. Indicates that the GLB interface requires attention from the host. 872 31:31 glb => bool; 873 } 874 875 /// Clear job interrupts. Write only. 876 /// 877 /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. 878 pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { 879 /// Clear CSG request interrupts. 880 30:0 csg; 881 /// Clear GLB request interrupt. 882 31:31 glb => bool; 883 } 884 885 /// Mask for job interrupts. 886 /// 887 /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. 888 pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { 889 /// Enable CSG request interrupts. 890 30:0 csg; 891 /// Enable GLB request interrupt. 892 31:31 glb => bool; 893 } 894 895 /// Active job interrupts. Read only. 896 /// 897 /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and 898 /// [`JOB_IRQ_MASK`]. 899 pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { 900 /// CSG request interrupt status. 901 30:0 csg; 902 /// GLB request interrupt status. 903 31:31 glb => bool; 904 } 905 } 906 } 907 908 /// These registers correspond to the MMU_CONTROL register page. 909 /// They are involved in MMU configuration and control. 910 pub(crate) mod mmu_control { 911 use kernel::register; 912 913 register! { 914 /// IRQ sources raw status. 915 /// 916 /// This register contains the raw unmasked interrupt sources for MMU status and exception 917 /// handling. 918 /// 919 /// Writing to this register forces bits on. 920 /// Use [`IRQ_CLEAR`] to clear interrupts. 921 pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { 922 /// Page fault for address spaces. 923 15:0 page_fault; 924 /// Command completed in address spaces. 925 31:16 command_completed; 926 } 927 928 /// IRQ sources to clear. 929 /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. 930 pub(crate) IRQ_CLEAR(u32) @ 0x2004 { 931 /// Clear the PAGE_FAULT interrupt. 932 15:0 page_fault; 933 /// Clear the COMMAND_COMPLETED interrupt. 934 31:16 command_completed; 935 } 936 937 /// IRQ sources enabled. 938 /// 939 /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. 940 pub(crate) IRQ_MASK(u32) @ 0x2008 { 941 /// Enable the PAGE_FAULT interrupt. 942 15:0 page_fault; 943 /// Enable the COMMAND_COMPLETED interrupt. 944 31:16 command_completed; 945 } 946 947 /// IRQ status for enabled sources. Read only. 948 /// 949 /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. 950 pub(crate) IRQ_STATUS(u32) @ 0x200c { 951 /// PAGE_FAULT interrupt status. 952 15:0 page_fault; 953 /// COMMAND_COMPLETED interrupt status. 954 31:16 command_completed; 955 } 956 } 957 958 /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. 959 /// 960 /// This array contains 16 instances of the MMU_AS_CONTROL register page. 961 pub(crate) mod mmu_as_control { 962 use kernel::{ 963 num::Bounded, 964 prelude::*, 965 register, // 966 }; 967 968 /// Maximum number of hardware address space slots. 969 /// The actual number of slots available is usually lower. 970 pub(crate) const MAX_AS: usize = 16; 971 972 /// Address space register stride. The elements in the array are spaced 64B apart. 973 const STRIDE: usize = 0x40; 974 975 register! { 976 /// Translation table base address. A 64-bit pointer. 977 /// 978 /// This field contains the address of the top level of a translation table structure. 979 /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. 980 pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { 981 /// Base address of the translation table. 982 63:0 base; 983 } 984 985 // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two 986 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 987 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 988 pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { 989 31:0 value; 990 } 991 992 pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { 993 31:0 value; 994 } 995 } 996 997 /// Helpers for MEMATTR Register. 998 999 #[derive(Copy, Clone, Debug, PartialEq)] 1000 #[repr(u8)] 1001 pub(crate) enum AllocPolicySelect { 1002 /// Ignore ALLOC_R/ALLOC_W fields. 1003 Impl = 2, 1004 /// Use ALLOC_R/ALLOC_W fields for allocation policy. 1005 Alloc = 3, 1006 } 1007 1008 impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect { 1009 type Error = Error; 1010 1011 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1012 match val.get() { 1013 2 => Ok(Self::Impl), 1014 3 => Ok(Self::Alloc), 1015 _ => Err(EINVAL), 1016 } 1017 } 1018 } 1019 1020 impl From<AllocPolicySelect> for Bounded<u8, 2> { 1021 fn from(val: AllocPolicySelect) -> Self { 1022 Bounded::try_new(val as u8).unwrap() 1023 } 1024 } 1025 1026 /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. 1027 /// 1028 /// The hardware spec defines different interpretations of these values depending on 1029 /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation 1030 /// tables (all input addresses map to the same output address); it is deprecated and not 1031 /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. 1032 #[derive(Copy, Clone, Debug, PartialEq)] 1033 #[repr(u8)] 1034 pub(crate) enum Coherency { 1035 /// Midgard inner domain coherency. 1036 /// 1037 /// Most flexible mode - can map non-coherent, internally coherent, and system/IO 1038 /// coherent memory. Used for non-cacheable memory in MAIR conversion. 1039 MidgardInnerDomain = 0, 1040 /// CPU inner domain coherency. 1041 /// 1042 /// Can map non-coherent and system/IO coherent memory. Used for write-back 1043 /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. 1044 CpuInnerDomain = 1, 1045 /// CPU inner domain with shader coherency. 1046 /// 1047 /// Can map internally coherent and system/IO coherent memory. Used for 1048 /// GPU-internal shared buffers requiring shader coherency. 1049 CpuInnerDomainShaderCoh = 2, 1050 } 1051 1052 impl TryFrom<Bounded<u8, 2>> for Coherency { 1053 type Error = Error; 1054 1055 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1056 match val.get() { 1057 0 => Ok(Self::MidgardInnerDomain), 1058 1 => Ok(Self::CpuInnerDomain), 1059 2 => Ok(Self::CpuInnerDomainShaderCoh), 1060 _ => Err(EINVAL), 1061 } 1062 } 1063 } 1064 1065 impl From<Coherency> for Bounded<u8, 2> { 1066 fn from(val: Coherency) -> Self { 1067 Bounded::try_new(val as u8).unwrap() 1068 } 1069 } 1070 1071 #[derive(Copy, Clone, Debug, PartialEq)] 1072 #[repr(u8)] 1073 pub(crate) enum MemoryType { 1074 /// Normal memory (shared). 1075 Shared = 0, 1076 /// Normal memory, inner/outer non-cacheable. 1077 NonCacheable = 1, 1078 /// Normal memory, inner/outer write-back cacheable. 1079 WriteBack = 2, 1080 /// Triggers MEMORY_ATTRIBUTE_FAULT. 1081 Fault = 3, 1082 } 1083 1084 impl From<Bounded<u8, 2>> for MemoryType { 1085 fn from(val: Bounded<u8, 2>) -> Self { 1086 match val.get() { 1087 0 => Self::Shared, 1088 1 => Self::NonCacheable, 1089 2 => Self::WriteBack, 1090 3 => Self::Fault, 1091 _ => unreachable!(), 1092 } 1093 } 1094 } 1095 1096 impl From<MemoryType> for Bounded<u8, 2> { 1097 fn from(val: MemoryType) -> Self { 1098 Bounded::try_new(val as u8).unwrap() 1099 } 1100 } 1101 1102 register! { 1103 /// Stage 1 memory attributes (8-bit bitfield). 1104 /// 1105 /// This is not an actual register, but a bitfield definition used by the MEMATTR 1106 /// register. Each of the 8 bytes in MEMATTR follows this layout. 1107 MMU_MEMATTR_STAGE1(u8) @ 0x0 { 1108 /// Inner cache write allocation policy. 1109 0:0 alloc_w => bool; 1110 /// Inner cache read allocation policy. 1111 1:1 alloc_r => bool; 1112 /// Inner allocation policy select. 1113 3:2 alloc_sel ?=> AllocPolicySelect; 1114 /// Coherency policy. 1115 5:4 coherency ?=> Coherency; 1116 /// Memory type. 1117 7:6 memory_type => MemoryType; 1118 } 1119 } 1120 1121 impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 { 1122 type Error = Error; 1123 1124 fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> { 1125 Ok(Self::from_raw(val.get() as u8)) 1126 } 1127 } 1128 1129 impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> { 1130 fn from(val: MMU_MEMATTR_STAGE1) -> Self { 1131 Bounded::try_new(u64::from(val.into_raw())).unwrap() 1132 } 1133 } 1134 1135 register! { 1136 /// Memory attributes. 1137 /// 1138 /// Each address space can configure up to 8 different memory attribute profiles. 1139 /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. 1140 pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { 1141 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; 1142 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; 1143 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; 1144 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; 1145 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; 1146 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; 1147 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; 1148 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; 1149 } 1150 1151 // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two 1152 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1153 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1154 pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { 1155 31:0 value; 1156 } 1157 1158 pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { 1159 31:0 value; 1160 } 1161 1162 /// Lock region address for each address space. 1163 pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { 1164 /// Lock region size. 1165 5:0 size; 1166 /// Lock region base address. 1167 63:12 base; 1168 } 1169 1170 // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two 1171 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1172 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1173 pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { 1174 31:0 value; 1175 } 1176 1177 pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { 1178 31:0 value; 1179 } 1180 } 1181 1182 /// Helpers for MMU COMMAND register. 1183 #[derive(Copy, Clone, Debug, PartialEq)] 1184 #[repr(u8)] 1185 pub(crate) enum MmuCommand { 1186 /// No operation, nothing happens. 1187 Nop = 0, 1188 /// Propagate settings to the MMU. 1189 Update = 1, 1190 /// Lock an address region. 1191 Lock = 2, 1192 /// Unlock an address region. 1193 Unlock = 3, 1194 /// Clean and invalidate the L2 cache, then unlock. 1195 FlushPt = 4, 1196 /// Clean and invalidate all caches, then unlock. 1197 FlushMem = 5, 1198 } 1199 1200 impl TryFrom<Bounded<u32, 8>> for MmuCommand { 1201 type Error = Error; 1202 1203 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1204 match val.get() { 1205 0 => Ok(MmuCommand::Nop), 1206 1 => Ok(MmuCommand::Update), 1207 2 => Ok(MmuCommand::Lock), 1208 3 => Ok(MmuCommand::Unlock), 1209 4 => Ok(MmuCommand::FlushPt), 1210 5 => Ok(MmuCommand::FlushMem), 1211 _ => Err(EINVAL), 1212 } 1213 } 1214 } 1215 1216 impl From<MmuCommand> for Bounded<u32, 8> { 1217 fn from(cmd: MmuCommand) -> Self { 1218 (cmd as u8).into() 1219 } 1220 } 1221 1222 register! { 1223 /// MMU command register for each address space. Write only. 1224 pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 1225 7:0 command ?=> MmuCommand; 1226 } 1227 } 1228 1229 /// MMU exception types for FAULTSTATUS register. 1230 #[derive(Copy, Clone, Debug, PartialEq)] 1231 #[repr(u8)] 1232 pub(crate) enum MmuExceptionType { 1233 /// No error. 1234 Ok = 0x00, 1235 /// Invalid translation table entry, level 0. 1236 TranslationFault0 = 0xC0, 1237 /// Invalid translation table entry, level 1. 1238 TranslationFault1 = 0xC1, 1239 /// Invalid translation table entry, level 2. 1240 TranslationFault2 = 0xC2, 1241 /// Invalid translation table entry, level 3. 1242 TranslationFault3 = 0xC3, 1243 /// Invalid block descriptor. 1244 TranslationFault4 = 0xC4, 1245 /// Page permission error, level 0. 1246 PermissionFault0 = 0xC8, 1247 /// Page permission error, level 1. 1248 PermissionFault1 = 0xC9, 1249 /// Page permission error, level 2. 1250 PermissionFault2 = 0xCA, 1251 /// Page permission error, level 3. 1252 PermissionFault3 = 0xCB, 1253 /// Access flag not set, level 1. 1254 AccessFlag1 = 0xD9, 1255 /// Access flag not set, level 2. 1256 AccessFlag2 = 0xDA, 1257 /// Access flag not set, level 3. 1258 AccessFlag3 = 0xDB, 1259 /// Virtual address out of range. 1260 AddressSizeFaultIn = 0xE0, 1261 /// Physical address out of range, level 0. 1262 AddressSizeFaultOut0 = 0xE4, 1263 /// Physical address out of range, level 1. 1264 AddressSizeFaultOut1 = 0xE5, 1265 /// Physical address out of range, level 2. 1266 AddressSizeFaultOut2 = 0xE6, 1267 /// Physical address out of range, level 3. 1268 AddressSizeFaultOut3 = 0xE7, 1269 /// Page attribute error, level 0. 1270 MemoryAttributeFault0 = 0xE8, 1271 /// Page attribute error, level 1. 1272 MemoryAttributeFault1 = 0xE9, 1273 /// Page attribute error, level 2. 1274 MemoryAttributeFault2 = 0xEA, 1275 /// Page attribute error, level 3. 1276 MemoryAttributeFault3 = 0xEB, 1277 } 1278 1279 impl TryFrom<Bounded<u32, 8>> for MmuExceptionType { 1280 type Error = Error; 1281 1282 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1283 match val.get() { 1284 0x00 => Ok(MmuExceptionType::Ok), 1285 0xC0 => Ok(MmuExceptionType::TranslationFault0), 1286 0xC1 => Ok(MmuExceptionType::TranslationFault1), 1287 0xC2 => Ok(MmuExceptionType::TranslationFault2), 1288 0xC3 => Ok(MmuExceptionType::TranslationFault3), 1289 0xC4 => Ok(MmuExceptionType::TranslationFault4), 1290 0xC8 => Ok(MmuExceptionType::PermissionFault0), 1291 0xC9 => Ok(MmuExceptionType::PermissionFault1), 1292 0xCA => Ok(MmuExceptionType::PermissionFault2), 1293 0xCB => Ok(MmuExceptionType::PermissionFault3), 1294 0xD9 => Ok(MmuExceptionType::AccessFlag1), 1295 0xDA => Ok(MmuExceptionType::AccessFlag2), 1296 0xDB => Ok(MmuExceptionType::AccessFlag3), 1297 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), 1298 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), 1299 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), 1300 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), 1301 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), 1302 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), 1303 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), 1304 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), 1305 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), 1306 _ => Err(EINVAL), 1307 } 1308 } 1309 } 1310 1311 impl From<MmuExceptionType> for Bounded<u32, 8> { 1312 fn from(exc: MmuExceptionType) -> Self { 1313 (exc as u8).into() 1314 } 1315 } 1316 1317 /// Access type for MMU faults. 1318 #[derive(Copy, Clone, Debug, PartialEq)] 1319 #[repr(u8)] 1320 pub(crate) enum MmuAccessType { 1321 /// An atomic (read/write) transaction. 1322 Atomic = 0, 1323 /// An execute transaction. 1324 Execute = 1, 1325 /// A read transaction. 1326 Read = 2, 1327 /// A write transaction. 1328 Write = 3, 1329 } 1330 1331 impl From<Bounded<u32, 2>> for MmuAccessType { 1332 fn from(val: Bounded<u32, 2>) -> Self { 1333 match val.get() { 1334 0 => MmuAccessType::Atomic, 1335 1 => MmuAccessType::Execute, 1336 2 => MmuAccessType::Read, 1337 3 => MmuAccessType::Write, 1338 _ => unreachable!(), 1339 } 1340 } 1341 } 1342 1343 impl From<MmuAccessType> for Bounded<u32, 2> { 1344 fn from(access: MmuAccessType) -> Self { 1345 Bounded::try_new(access as u32).unwrap() 1346 } 1347 } 1348 1349 register! { 1350 /// Fault status register for each address space. Read only. 1351 pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { 1352 /// Exception type. 1353 7:0 exception_type ?=> MmuExceptionType; 1354 /// Access type. 1355 9:8 access_type => MmuAccessType; 1356 /// ID of the source that triggered the fault. 1357 31:16 source_id; 1358 } 1359 1360 /// Fault address for each address space. Read only. 1361 pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { 1362 31:0 pointer; 1363 } 1364 1365 pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { 1366 31:0 pointer; 1367 } 1368 1369 /// MMU status register for each address space. Read only. 1370 pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { 1371 /// External address space command is active, a 1-bit boolean flag. 1372 0:0 active_ext => bool; 1373 /// Internal address space command is active, a 1-bit boolean flag. 1374 1:1 active_int => bool; 1375 } 1376 } 1377 1378 /// Helpers for TRANSCFG register. 1379 /// 1380 /// Address space mode for TRANSCFG register. 1381 #[derive(Copy, Clone, Debug, PartialEq)] 1382 #[repr(u8)] 1383 pub(crate) enum AddressSpaceMode { 1384 /// The MMU forces all memory access to fail with a decode fault. 1385 Unmapped = 1, 1386 /// All input addresses map to the same output address (deprecated). 1387 Identity = 2, 1388 /// Translation tables interpreted according to AArch64 4kB granule specification. 1389 Aarch64_4K = 6, 1390 /// Translation tables interpreted according to AArch64 64kB granule specification. 1391 Aarch64_64K = 8, 1392 } 1393 1394 impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode { 1395 type Error = Error; 1396 1397 fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> { 1398 match val.get() { 1399 1 => Ok(AddressSpaceMode::Unmapped), 1400 2 => Ok(AddressSpaceMode::Identity), 1401 6 => Ok(AddressSpaceMode::Aarch64_4K), 1402 8 => Ok(AddressSpaceMode::Aarch64_64K), 1403 _ => Err(EINVAL), 1404 } 1405 } 1406 } 1407 1408 impl From<AddressSpaceMode> for Bounded<u64, 4> { 1409 fn from(mode: AddressSpaceMode) -> Self { 1410 Bounded::try_new(mode as u64).unwrap() 1411 } 1412 } 1413 1414 /// Input address range restriction for TRANSCFG register. 1415 #[derive(Copy, Clone, Debug, PartialEq)] 1416 #[repr(u8)] 1417 pub(crate) enum InaBits { 1418 /// Invalid VA range (reset value). 1419 Reset = 0, 1420 /// 48-bit VA range. 1421 Bits48 = 7, 1422 /// 47-bit VA range. 1423 Bits47 = 8, 1424 /// 46-bit VA range. 1425 Bits46 = 9, 1426 /// 45-bit VA range. 1427 Bits45 = 10, 1428 /// 44-bit VA range. 1429 Bits44 = 11, 1430 /// 43-bit VA range. 1431 Bits43 = 12, 1432 /// 42-bit VA range. 1433 Bits42 = 13, 1434 /// 41-bit VA range. 1435 Bits41 = 14, 1436 /// 40-bit VA range. 1437 Bits40 = 15, 1438 /// 39-bit VA range. 1439 Bits39 = 16, 1440 /// 38-bit VA range. 1441 Bits38 = 17, 1442 /// 37-bit VA range. 1443 Bits37 = 18, 1444 /// 36-bit VA range. 1445 Bits36 = 19, 1446 /// 35-bit VA range. 1447 Bits35 = 20, 1448 /// 34-bit VA range. 1449 Bits34 = 21, 1450 /// 33-bit VA range. 1451 Bits33 = 22, 1452 /// 32-bit VA range. 1453 Bits32 = 23, 1454 /// 31-bit VA range. 1455 Bits31 = 24, 1456 /// 30-bit VA range. 1457 Bits30 = 25, 1458 /// 29-bit VA range. 1459 Bits29 = 26, 1460 /// 28-bit VA range. 1461 Bits28 = 27, 1462 /// 27-bit VA range. 1463 Bits27 = 28, 1464 /// 26-bit VA range. 1465 Bits26 = 29, 1466 /// 25-bit VA range. 1467 Bits25 = 30, 1468 } 1469 1470 impl TryFrom<Bounded<u64, 5>> for InaBits { 1471 type Error = Error; 1472 1473 fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> { 1474 match val.get() { 1475 0 => Ok(InaBits::Reset), 1476 7 => Ok(InaBits::Bits48), 1477 8 => Ok(InaBits::Bits47), 1478 9 => Ok(InaBits::Bits46), 1479 10 => Ok(InaBits::Bits45), 1480 11 => Ok(InaBits::Bits44), 1481 12 => Ok(InaBits::Bits43), 1482 13 => Ok(InaBits::Bits42), 1483 14 => Ok(InaBits::Bits41), 1484 15 => Ok(InaBits::Bits40), 1485 16 => Ok(InaBits::Bits39), 1486 17 => Ok(InaBits::Bits38), 1487 18 => Ok(InaBits::Bits37), 1488 19 => Ok(InaBits::Bits36), 1489 20 => Ok(InaBits::Bits35), 1490 21 => Ok(InaBits::Bits34), 1491 22 => Ok(InaBits::Bits33), 1492 23 => Ok(InaBits::Bits32), 1493 24 => Ok(InaBits::Bits31), 1494 25 => Ok(InaBits::Bits30), 1495 26 => Ok(InaBits::Bits29), 1496 27 => Ok(InaBits::Bits28), 1497 28 => Ok(InaBits::Bits27), 1498 29 => Ok(InaBits::Bits26), 1499 30 => Ok(InaBits::Bits25), 1500 _ => Err(EINVAL), 1501 } 1502 } 1503 } 1504 1505 impl From<InaBits> for Bounded<u64, 5> { 1506 fn from(bits: InaBits) -> Self { 1507 Bounded::try_new(bits as u64).unwrap() 1508 } 1509 } 1510 1511 /// Translation table memory attributes for TRANSCFG register. 1512 #[derive(Copy, Clone, Debug, PartialEq)] 1513 #[repr(u8)] 1514 pub(crate) enum PtwMemattr { 1515 /// Invalid (reset value, not valid for enabled address space). 1516 Invalid = 0, 1517 /// Normal memory, inner/outer non-cacheable. 1518 NonCacheable = 1, 1519 /// Normal memory, inner/outer write-back cacheable. 1520 WriteBack = 2, 1521 } 1522 1523 impl TryFrom<Bounded<u64, 2>> for PtwMemattr { 1524 type Error = Error; 1525 1526 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1527 match val.get() { 1528 0 => Ok(PtwMemattr::Invalid), 1529 1 => Ok(PtwMemattr::NonCacheable), 1530 2 => Ok(PtwMemattr::WriteBack), 1531 _ => Err(EINVAL), 1532 } 1533 } 1534 } 1535 1536 impl From<PtwMemattr> for Bounded<u64, 2> { 1537 fn from(attr: PtwMemattr) -> Self { 1538 Bounded::try_new(attr as u64).unwrap() 1539 } 1540 } 1541 1542 /// Translation table memory shareability for TRANSCFG register. 1543 #[derive(Copy, Clone, Debug, PartialEq)] 1544 #[repr(u8)] 1545 #[allow(clippy::enum_variant_names)] 1546 pub(crate) enum PtwShareability { 1547 /// Non-shareable. 1548 NonShareable = 0, 1549 /// Outer shareable. 1550 OuterShareable = 2, 1551 /// Inner shareable. 1552 InnerShareable = 3, 1553 } 1554 1555 impl TryFrom<Bounded<u64, 2>> for PtwShareability { 1556 type Error = Error; 1557 1558 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1559 match val.get() { 1560 0 => Ok(PtwShareability::NonShareable), 1561 2 => Ok(PtwShareability::OuterShareable), 1562 3 => Ok(PtwShareability::InnerShareable), 1563 _ => Err(EINVAL), 1564 } 1565 } 1566 } 1567 1568 impl From<PtwShareability> for Bounded<u64, 2> { 1569 fn from(sh: PtwShareability) -> Self { 1570 Bounded::try_new(sh as u64).unwrap() 1571 } 1572 } 1573 1574 register! { 1575 /// Translation configuration and control. 1576 pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { 1577 /// Address space mode. 1578 3:0 mode ?=> AddressSpaceMode; 1579 /// Address input restriction. 1580 10:6 ina_bits ?=> InaBits; 1581 /// Address output restriction. 1582 18:14 outa_bits; 1583 /// Translation table concatenation enable, a 1-bit boolean flag. 1584 22:22 sl_concat_en => bool; 1585 /// Translation table memory attributes. 1586 25:24 ptw_memattr ?=> PtwMemattr; 1587 /// Translation table memory shareability. 1588 29:28 ptw_sh ?=> PtwShareability; 1589 /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. 1590 30:30 r_allocate => bool; 1591 /// Disable hierarchical access permissions. 1592 33:33 disable_hier_ap => bool; 1593 /// Disable access fault checking. 1594 34:34 disable_af_fault => bool; 1595 /// Disable execution on all writable pages. 1596 35:35 wxn => bool; 1597 /// Enable execution on readable pages. 1598 36:36 xreadable => bool; 1599 /// Page-based hardware attributes for translation table walks. 1600 63:60 ptw_pbha; 1601 } 1602 1603 // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two 1604 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1605 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1606 pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { 1607 31:0 value; 1608 } 1609 1610 pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { 1611 31:0 value; 1612 } 1613 1614 /// Extra fault information for each address space. Read only. 1615 pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { 1616 31:0 value; 1617 } 1618 1619 pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { 1620 31:0 value; 1621 } 1622 } 1623 } 1624 } 1625 1626 /// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. 1627 pub(crate) mod doorbell_block { 1628 use kernel::register; 1629 1630 /// Number of doorbells available. 1631 pub(crate) const NUM_DOORBELLS: usize = 64; 1632 1633 /// Doorbell block stride (64KiB). 1634 /// 1635 /// Each block occupies a full page, allowing it to be mapped 1636 /// separately into a virtual address space. 1637 const STRIDE: usize = 0x10000; 1638 1639 register! { 1640 /// Doorbell request register. Write-only. 1641 pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { 1642 /// Doorbell set. Writing 1 triggers the doorbell. 1643 0:0 ring => bool; 1644 } 1645 } 1646 } 1647