1 // SPDX-License-Identifier: GPL-2.0 or MIT 2 3 //! # Definitions 4 //! 5 //! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions) 6 //! from the command stream. 7 //! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular 8 //! job or sequence of jobs. The instructions exist in one or more command buffers. 9 //! - **CSF**: Command Stream Frontend - The interface and implementation for job submission 10 //! exposed to the host CPU driver. This includes the global interface, as well as CSG and CS 11 //! interfaces. 12 //! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple 13 //! CSGs, and each CSG contains multiple CSs. 14 //! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the 15 //! iterator control aspects. Implements the CSF in conjunction with the MCU. 16 //! - **GLB**: Global - Prefix for global interface registers that control operations common to 17 //! all CSs. 18 //! - **JASID**: Job Address Space ID - Identifies the address space for a job. 19 //! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream 20 //! hardware. 21 //! - **MMU**: Memory Management Unit - Handles address translation and memory access protection. 22 23 // We don't expect that all the registers and fields will be used, even in the 24 // future. 25 // 26 // Nevertheless, it is useful to have most of them defined, like the C driver 27 // does. 28 #![allow(dead_code)] 29 30 /// Combine two 32-bit values into a single 64-bit value. 31 pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 { 32 (u64::from(lo)) | ((u64::from(hi)) << 32) 33 } 34 35 /// Read a logical 64-bit value from split 32-bit registers without tearing. 36 pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 { 37 loop { 38 let hi1 = hi_read(); 39 let lo = lo_read(); 40 let hi2 = hi_read(); 41 42 if hi1 == hi2 { 43 return join_u64(lo, hi1); 44 } 45 } 46 } 47 48 /// These registers correspond to the GPU_CONTROL register page. 49 /// They are involved in GPU configuration and control. 50 pub(crate) mod gpu_control { 51 use core::convert::TryFrom; 52 use kernel::{ 53 error::{ 54 code::EINVAL, 55 Error, // 56 }, 57 num::Bounded, 58 register, 59 uapi, // 60 }; 61 use pin_init::Zeroable; 62 63 register! { 64 /// GPU identification register. 65 pub(crate) GPU_ID(u32) @ 0x0 { 66 /// Status of the GPU release. 67 3:0 ver_status; 68 /// Minor release version number. 69 11:4 ver_minor; 70 /// Major release version number. 71 15:12 ver_major; 72 /// Product identifier. 73 19:16 prod_major; 74 /// Architecture patch revision. 75 23:20 arch_rev; 76 /// Architecture minor revision. 77 27:24 arch_minor; 78 /// Architecture major revision. 79 31:28 arch_major; 80 } 81 82 /// Level 2 cache features register. 83 pub(crate) L2_FEATURES(u32) @ 0x4 { 84 /// Cache line size. 85 7:0 line_size; 86 /// Cache associativity. 87 15:8 associativity; 88 /// Cache slice size. 89 23:16 cache_size; 90 /// External bus width. 91 31:24 bus_width; 92 } 93 94 /// Shader core features. 95 pub(crate) CORE_FEATURES(u32) @ 0x8 { 96 /// Shader core variant. 97 7:0 core_variant; 98 } 99 100 /// Tiler features. 101 pub(crate) TILER_FEATURES(u32) @ 0xc { 102 /// Log of the tiler's bin size. 103 5:0 bin_size; 104 /// Maximum number of active levels. 105 11:8 max_levels; 106 } 107 108 /// Memory system features. 109 pub(crate) MEM_FEATURES(u32) @ 0x10 { 110 0:0 coherent_core_group => bool; 111 1:1 coherent_super_group => bool; 112 11:8 l2_slices; 113 } 114 115 /// Memory management unit features. 116 pub(crate) MMU_FEATURES(u32) @ 0x14 { 117 /// Number of bits supported in virtual addresses. 118 7:0 va_bits; 119 /// Number of bits supported in physical addresses. 120 15:8 pa_bits; 121 } 122 123 /// Address spaces present. 124 pub(crate) AS_PRESENT(u32) @ 0x18 { 125 31:0 present; 126 } 127 128 /// CSF version information. 129 pub(crate) CSF_ID(u32) @ 0x1c { 130 /// MCU revision ID. 131 3:0 mcu_rev; 132 /// MCU minor revision number. 133 9:4 mcu_minor; 134 /// MCU major revision number. 135 15:10 mcu_major; 136 /// CSHW revision ID. 137 19:16 cshw_rev; 138 /// CSHW minor revision number. 139 25:20 cshw_minor; 140 /// CSHW major revision number. 141 31:26 cshw_major; 142 } 143 144 /// IRQ sources raw status. 145 /// Writing to this register forces bits on, but does not clear them. 146 pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 { 147 /// A GPU fault has occurred, a 1-bit boolean flag. 148 0:0 gpu_fault => bool; 149 /// A GPU fault has occurred, a 1-bit boolean flag. 150 1:1 gpu_protected_fault => bool; 151 /// Reset has completed, a 1-bit boolean flag. 152 8:8 reset_completed => bool; 153 /// Set when a single power domain has powered up or down, a 1-bit boolean flag. 154 9:9 power_changed_single => bool; 155 /// Set when the all pending power domain changes are completed, a 1-bit boolean flag. 156 10:10 power_changed_all => bool; 157 /// Set when cache cleaning has completed, a 1-bit boolean flag. 158 17:17 clean_caches_completed => bool; 159 /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag. 160 18:18 doorbell_mirror => bool; 161 /// MCU requires attention, a 1-bit boolean flag. 162 19:19 mcu_status => bool; 163 } 164 165 /// IRQ sources to clear. Write only. 166 pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 { 167 /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag. 168 0:0 gpu_fault => bool; 169 /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 170 1:1 gpu_protected_fault => bool; 171 /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag. 172 8:8 reset_completed => bool; 173 /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 174 9:9 power_changed_single => bool; 175 /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 176 10:10 power_changed_all => bool; 177 /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 178 17:17 clean_caches_completed => bool; 179 /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag. 180 19:19 mcu_status => bool; 181 } 182 183 /// IRQ sources enabled. 184 pub(crate) GPU_IRQ_MASK(u32) @ 0x28 { 185 /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag. 186 0:0 gpu_fault => bool; 187 /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag. 188 1:1 gpu_protected_fault => bool; 189 /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag. 190 8:8 reset_completed => bool; 191 /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag. 192 9:9 power_changed_single => bool; 193 /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag. 194 10:10 power_changed_all => bool; 195 /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag. 196 17:17 clean_caches_completed => bool; 197 /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag. 198 18:18 doorbell_mirror => bool; 199 /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag. 200 19:19 mcu_status => bool; 201 } 202 203 /// IRQ status for enabled sources. Read only. 204 pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c { 205 /// GPU_FAULT interrupt status, a 1-bit boolean flag. 206 0:0 gpu_fault => bool; 207 /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag. 208 1:1 gpu_protected_fault => bool; 209 /// RESET_COMPLETED interrupt status, a 1-bit boolean flag. 210 8:8 reset_completed => bool; 211 /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag. 212 9:9 power_changed_single => bool; 213 /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag. 214 10:10 power_changed_all => bool; 215 /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag. 216 17:17 clean_caches_completed => bool; 217 /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag. 218 18:18 doorbell_mirror => bool; 219 /// MCU_STATUS interrupt status, a 1-bit boolean flag. 220 19:19 mcu_status => bool; 221 } 222 } 223 224 /// Helpers for GPU_COMMAND Register 225 #[derive(Copy, Clone, Debug, PartialEq)] 226 #[repr(u8)] 227 pub(crate) enum GpuCommand { 228 /// No operation. This is the default value. 229 Nop = 0, 230 /// Reset the GPU. 231 Reset = 1, 232 /// Flush caches. 233 FlushCaches = 4, 234 /// Clear GPU faults. 235 ClearFault = 7, 236 } 237 238 impl TryFrom<Bounded<u32, 8>> for GpuCommand { 239 type Error = Error; 240 241 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 242 match val.get() { 243 0 => Ok(GpuCommand::Nop), 244 1 => Ok(GpuCommand::Reset), 245 4 => Ok(GpuCommand::FlushCaches), 246 7 => Ok(GpuCommand::ClearFault), 247 _ => Err(EINVAL), 248 } 249 } 250 } 251 252 impl From<GpuCommand> for Bounded<u32, 8> { 253 fn from(cmd: GpuCommand) -> Self { 254 (cmd as u8).into() 255 } 256 } 257 258 /// Reset mode for [`GPU_COMMAND::reset()`]. 259 #[derive(Copy, Clone, Debug, PartialEq)] 260 #[repr(u8)] 261 pub(crate) enum ResetMode { 262 /// Stop all external bus interfaces, then reset the entire GPU. 263 SoftReset = 1, 264 /// Force a full GPU reset. 265 HardReset = 2, 266 } 267 268 impl TryFrom<Bounded<u32, 4>> for ResetMode { 269 type Error = Error; 270 271 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 272 match val.get() { 273 1 => Ok(ResetMode::SoftReset), 274 2 => Ok(ResetMode::HardReset), 275 _ => Err(EINVAL), 276 } 277 } 278 } 279 280 impl From<ResetMode> for Bounded<u32, 4> { 281 fn from(mode: ResetMode) -> Self { 282 Bounded::try_new(mode as u32).unwrap() 283 } 284 } 285 286 /// Cache flush mode for [`GPU_COMMAND::flush_caches()`]. 287 #[derive(Copy, Clone, Debug, PartialEq)] 288 #[repr(u8)] 289 pub(crate) enum FlushMode { 290 /// No flush. 291 None = 0, 292 /// Clean the caches. 293 Clean = 1, 294 /// Invalidate the caches. 295 Invalidate = 2, 296 /// Clean and invalidate the caches. 297 CleanInvalidate = 3, 298 } 299 300 impl TryFrom<Bounded<u32, 4>> for FlushMode { 301 type Error = Error; 302 303 fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> { 304 match val.get() { 305 0 => Ok(FlushMode::None), 306 1 => Ok(FlushMode::Clean), 307 2 => Ok(FlushMode::Invalidate), 308 3 => Ok(FlushMode::CleanInvalidate), 309 _ => Err(EINVAL), 310 } 311 } 312 } 313 314 impl From<FlushMode> for Bounded<u32, 4> { 315 fn from(mode: FlushMode) -> Self { 316 Bounded::try_new(mode as u32).unwrap() 317 } 318 } 319 320 register! { 321 /// GPU command register. 322 /// 323 /// Use the constructor methods to create commands: 324 /// - [`GPU_COMMAND::nop()`] 325 /// - [`GPU_COMMAND::reset()`] 326 /// - [`GPU_COMMAND::flush_caches()`] 327 /// - [`GPU_COMMAND::clear_fault()`] 328 pub(crate) GPU_COMMAND (u32) @ 0x30 { 329 7:0 command ?=> GpuCommand; 330 } 331 /// Internal alias for GPU_COMMAND in reset mode. 332 /// Use [`GPU_COMMAND::reset()`] instead. 333 GPU_COMMAND_RESET (u32) => GPU_COMMAND { 334 7:0 command ?=> GpuCommand; 335 11:8 reset_mode ?=> ResetMode; 336 } 337 338 /// Internal alias for GPU_COMMAND in cache flush mode. 339 /// Use [`GPU_COMMAND::flush_caches()`] instead. 340 GPU_COMMAND_FLUSH (u32) => GPU_COMMAND { 341 7:0 command ?=> GpuCommand; 342 /// L2 cache flush mode. 343 11:8 l2_flush ?=> FlushMode; 344 /// Shader core load/store cache flush mode. 345 15:12 lsc_flush ?=> FlushMode; 346 /// Shader core other caches flush mode. 347 19:16 other_flush ?=> FlushMode; 348 } 349 } 350 351 impl GPU_COMMAND { 352 /// Create a NOP command. 353 pub(crate) fn nop() -> Self { 354 Self::zeroed() 355 } 356 357 /// Create a reset command with the specified reset mode. 358 pub(crate) fn reset(mode: ResetMode) -> Self { 359 Self::from_raw( 360 GPU_COMMAND_RESET::zeroed() 361 .with_command(GpuCommand::Reset) 362 .with_reset_mode(mode) 363 .into_raw(), 364 ) 365 } 366 367 /// Create a cache flush command with the specified flush modes. 368 pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self { 369 Self::from_raw( 370 GPU_COMMAND_FLUSH::zeroed() 371 .with_command(GpuCommand::FlushCaches) 372 .with_l2_flush(l2) 373 .with_lsc_flush(lsc) 374 .with_other_flush(other) 375 .into_raw(), 376 ) 377 } 378 379 /// Create a clear fault command. 380 pub(crate) fn clear_fault() -> Self { 381 Self::zeroed().with_command(GpuCommand::ClearFault) 382 } 383 } 384 385 register! { 386 /// GPU status register. Read only. 387 pub(crate) GPU_STATUS(u32) @ 0x34 { 388 /// GPU active, a 1-bit boolean flag. 389 0:0 gpu_active => bool; 390 /// Power manager active, a 1-bit boolean flag 391 1:1 pwr_active => bool; 392 /// Page fault active, a 1-bit boolean flag. 393 4:4 page_fault => bool; 394 /// Protected mode active, a 1-bit boolean flag. 395 7:7 protected_mode_active => bool; 396 /// Debug mode active, a 1-bit boolean flag. 397 8:8 gpu_dbg_enabled => bool; 398 } 399 } 400 401 #[derive(Copy, Clone, Debug, PartialEq)] 402 #[repr(u8)] 403 pub(crate) enum ExceptionType { 404 /// Exception type: No error. 405 Ok = 0x00, 406 /// Exception type: GPU external bus error. 407 GpuBusFault = 0x80, 408 /// Exception type: GPU shareability error. 409 GpuShareabilityFault = 0x88, 410 /// Exception type: System shareability error. 411 SystemShareabilityFault = 0x89, 412 /// Exception type: GPU cacheability error. 413 GpuCacheabilityFault = 0x8A, 414 } 415 416 impl TryFrom<Bounded<u32, 8>> for ExceptionType { 417 type Error = Error; 418 419 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 420 match val.get() { 421 0x00 => Ok(ExceptionType::Ok), 422 0x80 => Ok(ExceptionType::GpuBusFault), 423 0x88 => Ok(ExceptionType::GpuShareabilityFault), 424 0x89 => Ok(ExceptionType::SystemShareabilityFault), 425 0x8A => Ok(ExceptionType::GpuCacheabilityFault), 426 _ => Err(EINVAL), 427 } 428 } 429 } 430 431 impl From<ExceptionType> for Bounded<u32, 8> { 432 fn from(exc: ExceptionType) -> Self { 433 (exc as u8).into() 434 } 435 } 436 437 #[derive(Copy, Clone, Debug, PartialEq)] 438 #[repr(u8)] 439 pub(crate) enum AccessType { 440 /// Access type: An atomic (read/write) transaction. 441 Atomic = 0, 442 /// Access type: An execute transaction. 443 Execute = 1, 444 /// Access type: A read transaction. 445 Read = 2, 446 /// Access type: A write transaction. 447 Write = 3, 448 } 449 450 impl From<Bounded<u32, 2>> for AccessType { 451 fn from(val: Bounded<u32, 2>) -> Self { 452 match val.get() { 453 0 => AccessType::Atomic, 454 1 => AccessType::Execute, 455 2 => AccessType::Read, 456 3 => AccessType::Write, 457 _ => unreachable!(), 458 } 459 } 460 } 461 462 impl From<AccessType> for Bounded<u32, 2> { 463 fn from(access: AccessType) -> Self { 464 Bounded::try_new(access as u32).unwrap() 465 } 466 } 467 468 register! { 469 /// GPU fault status register. Read only. 470 pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { 471 /// Exception type. 472 7:0 exception_type ?=> ExceptionType; 473 /// Access type. 474 9:8 access_type => AccessType; 475 /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag. 476 10:10 address_valid => bool; 477 /// The JASID field is valid, a 1-bit boolean flag. 478 11:11 jasid_valid => bool; 479 /// JASID of the fault, if known. 480 15:12 jasid; 481 /// ID of the source that triggered the fault. 482 31:16 source_id; 483 } 484 485 /// GPU fault address. Read only. 486 /// Once a fault is reported, it must be manually cleared by issuing a 487 /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU 488 /// faults will be reported until the previous fault has been cleared. 489 pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 { 490 31:0 pointer; 491 } 492 493 pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 { 494 31:0 pointer; 495 } 496 497 /// Level 2 cache configuration. 498 pub(crate) L2_CONFIG(u32) @ 0x48 { 499 /// Requested cache size. 500 23:16 cache_size; 501 /// Requested hash function index. 502 31:24 hash_function; 503 } 504 505 /// Global time stamp offset. 506 pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 { 507 31:0 offset; 508 } 509 510 pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c { 511 31:0 offset; 512 } 513 514 /// GPU cycle counter. Read only. 515 pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 { 516 31:0 count; 517 } 518 519 pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 { 520 31:0 count; 521 } 522 523 /// Global time stamp. Read only. 524 pub(crate) TIMESTAMP_LO(u32) @ 0x98 { 525 31:0 timestamp; 526 } 527 528 pub(crate) TIMESTAMP_HI(u32) @ 0x9c { 529 31:0 timestamp; 530 } 531 532 /// Maximum number of threads per core. Read only constant. 533 pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 { 534 31:0 threads; 535 } 536 537 /// Maximum number of threads per workgroup. Read only constant. 538 pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 { 539 31:0 threads; 540 } 541 542 /// Maximum number of threads per barrier. Read only constant. 543 pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 { 544 31:0 threads; 545 } 546 547 /// Thread features. Read only constant. 548 pub(crate) THREAD_FEATURES(u32) @ 0xac { 549 /// Total number of registers per core. 550 21:0 max_registers; 551 /// Implementation technology type. 552 23:22 implementation_technology; 553 /// Maximum number of compute tasks waiting. 554 31:24 max_task_queue; 555 } 556 557 /// Support flags for compressed texture formats. Read only constant. 558 /// 559 /// A bitmap where each bit indicates support for a specific compressed texture format. 560 /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec). 561 pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 { 562 31:0 supported_formats; 563 } 564 565 /// Shader core present bitmap. Read only constant. 566 pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 { 567 31:0 value; 568 } 569 570 pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 { 571 31:0 value; 572 } 573 574 /// Tiler present bitmap. Read only constant. 575 pub(crate) TILER_PRESENT_LO(u32) @ 0x110 { 576 31:0 present; 577 } 578 579 pub(crate) TILER_PRESENT_HI(u32) @ 0x114 { 580 31:0 present; 581 } 582 583 /// L2 cache present bitmap. Read only constant. 584 pub(crate) L2_PRESENT_LO(u32) @ 0x120 { 585 31:0 present; 586 } 587 588 pub(crate) L2_PRESENT_HI(u32) @ 0x124 { 589 31:0 present; 590 } 591 592 /// Shader core ready bitmap. Read only. 593 pub(crate) SHADER_READY_LO(u32) @ 0x140 { 594 31:0 ready; 595 } 596 597 pub(crate) SHADER_READY_HI(u32) @ 0x144 { 598 31:0 ready; 599 } 600 601 /// Tiler ready bitmap. Read only. 602 pub(crate) TILER_READY_LO(u32) @ 0x150 { 603 31:0 ready; 604 } 605 606 pub(crate) TILER_READY_HI(u32) @ 0x154 { 607 31:0 ready; 608 } 609 610 /// L2 ready bitmap. Read only. 611 pub(crate) L2_READY_LO(u32) @ 0x160 { 612 31:0 ready; 613 } 614 615 pub(crate) L2_READY_HI(u32) @ 0x164 { 616 31:0 ready; 617 } 618 619 /// Shader core power up bitmap. 620 pub(crate) SHADER_PWRON_LO(u32) @ 0x180 { 621 31:0 request; 622 } 623 624 pub(crate) SHADER_PWRON_HI(u32) @ 0x184 { 625 31:0 request; 626 } 627 628 /// Tiler power up bitmap. 629 pub(crate) TILER_PWRON_LO(u32) @ 0x190 { 630 31:0 request; 631 } 632 633 pub(crate) TILER_PWRON_HI(u32) @ 0x194 { 634 31:0 request; 635 } 636 637 /// L2 power up bitmap. 638 pub(crate) L2_PWRON_LO(u32) @ 0x1a0 { 639 31:0 request; 640 } 641 642 pub(crate) L2_PWRON_HI(u32) @ 0x1a4 { 643 31:0 request; 644 } 645 646 /// Shader core power down bitmap. 647 pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 { 648 31:0 request; 649 } 650 651 pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 { 652 31:0 request; 653 } 654 655 /// Tiler power down bitmap. 656 pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 { 657 31:0 request; 658 } 659 660 pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 { 661 31:0 request; 662 } 663 664 /// L2 power down bitmap. 665 pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 { 666 31:0 request; 667 } 668 669 pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 { 670 31:0 request; 671 } 672 673 /// Shader core power transition bitmap. Read-only. 674 pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 { 675 31:0 changing; 676 } 677 678 pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 { 679 31:0 changing; 680 } 681 682 /// Tiler power transition bitmap. Read-only. 683 pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 { 684 31:0 changing; 685 } 686 687 pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 { 688 31:0 changing; 689 } 690 691 /// L2 power transition bitmap. Read-only. 692 pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 { 693 31:0 changing; 694 } 695 696 pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 { 697 31:0 changing; 698 } 699 700 /// Shader core active bitmap. Read-only. 701 pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 { 702 31:0 active; 703 } 704 705 pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 { 706 31:0 active; 707 } 708 709 /// Tiler active bitmap. Read-only. 710 pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 { 711 31:0 active; 712 } 713 714 pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 { 715 31:0 active; 716 } 717 718 /// L2 active bitmap. Read-only. 719 pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 { 720 31:0 active; 721 } 722 723 pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 { 724 31:0 active; 725 } 726 727 /// Revision ID. Read only constant. 728 pub(crate) REVIDR(u32) @ 0x280 { 729 31:0 revision; 730 } 731 732 /// Coherency features present. Read only constant. 733 /// Supported protocols on the interconnect between the GPU and the 734 /// system into which it is integrated. 735 pub(crate) COHERENCY_FEATURES(u32) @ 0x300 { 736 /// ACE-Lite protocol supported, a 1-bit boolean flag. 737 0:0 ace_lite => bool; 738 /// ACE protocol supported, a 1-bit boolean flag. 739 1:1 ace => bool; 740 } 741 } 742 743 #[derive(Copy, Clone, Debug, PartialEq)] 744 #[repr(u8)] 745 pub(crate) enum CoherencyMode { 746 /// ACE-Lite coherency protocol. 747 AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8, 748 /// ACE coherency protocol. 749 Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8, 750 /// No coherency protocol. 751 None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8, 752 } 753 754 impl TryFrom<Bounded<u32, 32>> for CoherencyMode { 755 type Error = Error; 756 757 fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> { 758 match val.get() { 759 0 => Ok(CoherencyMode::AceLite), 760 1 => Ok(CoherencyMode::Ace), 761 31 => Ok(CoherencyMode::None), 762 _ => Err(EINVAL), 763 } 764 } 765 } 766 767 impl From<CoherencyMode> for Bounded<u32, 32> { 768 fn from(mode: CoherencyMode) -> Self { 769 (mode as u8).into() 770 } 771 } 772 773 register! { 774 /// Coherency enable. An index of which coherency protocols should be used. 775 /// This register only selects the protocol for coherency messages on the 776 /// interconnect. This is not to enable or disable coherency controlled by MMU. 777 pub(crate) COHERENCY_ENABLE(u32) @ 0x304 { 778 31:0 l2_cache_protocol_select ?=> CoherencyMode; 779 } 780 } 781 782 /// Helpers for MCU_CONTROL register 783 #[derive(Copy, Clone, Debug, PartialEq)] 784 #[repr(u8)] 785 pub(crate) enum McuControlMode { 786 /// Disable the MCU. 787 Disable = 0, 788 /// Enable the MCU. 789 Enable = 1, 790 /// Enable the MCU to execute and automatically reboot after a fast reset. 791 Auto = 2, 792 } 793 794 impl TryFrom<Bounded<u32, 2>> for McuControlMode { 795 type Error = Error; 796 797 fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> { 798 match val.get() { 799 0 => Ok(McuControlMode::Disable), 800 1 => Ok(McuControlMode::Enable), 801 2 => Ok(McuControlMode::Auto), 802 _ => Err(EINVAL), 803 } 804 } 805 } 806 807 impl From<McuControlMode> for Bounded<u32, 2> { 808 fn from(mode: McuControlMode) -> Self { 809 Bounded::try_new(mode as u32).unwrap() 810 } 811 } 812 813 register! { 814 /// MCU control. 815 pub(crate) MCU_CONTROL(u32) @ 0x700 { 816 /// Request MCU state change. 817 1:0 req ?=> McuControlMode; 818 } 819 } 820 821 /// Helpers for MCU_STATUS register 822 #[derive(Copy, Clone, Debug, PartialEq)] 823 #[repr(u8)] 824 pub(crate) enum McuStatus { 825 /// MCU is disabled. 826 Disabled = 0, 827 /// MCU is enabled. 828 Enabled = 1, 829 /// The MCU has halted by itself in an orderly manner to enable the core group to be 830 /// powered down. 831 Halt = 2, 832 /// The MCU has encountered an error that prevents it from continuing. 833 Fatal = 3, 834 } 835 836 impl From<Bounded<u32, 2>> for McuStatus { 837 fn from(val: Bounded<u32, 2>) -> Self { 838 match val.get() { 839 0 => McuStatus::Disabled, 840 1 => McuStatus::Enabled, 841 2 => McuStatus::Halt, 842 3 => McuStatus::Fatal, 843 _ => unreachable!(), 844 } 845 } 846 } 847 848 impl From<McuStatus> for Bounded<u32, 2> { 849 fn from(status: McuStatus) -> Self { 850 Bounded::try_new(status as u32).unwrap() 851 } 852 } 853 854 register! { 855 /// MCU status. Read only. 856 pub(crate) MCU_STATUS(u32) @ 0x704 { 857 /// Read current state of MCU. 858 1:0 value => McuStatus; 859 } 860 } 861 } 862 863 /// These registers correspond to the JOB_CONTROL register page. 864 /// They are involved in communication between the firmware running on the MCU and the host. 865 pub(crate) mod job_control { 866 use kernel::register; 867 868 register! { 869 /// Raw status of job interrupts. 870 /// 871 /// Write to this register to trigger these interrupts. 872 /// Writing a 1 to a bit forces that bit on. 873 pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 { 874 /// CSG request. These bits indicate that CSGn requires attention from the host. 875 30:0 csg; 876 /// GLB request. Indicates that the GLB interface requires attention from the host. 877 31:31 glb => bool; 878 } 879 880 /// Clear job interrupts. Write only. 881 /// 882 /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`]. 883 pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 { 884 /// Clear CSG request interrupts. 885 30:0 csg; 886 /// Clear GLB request interrupt. 887 31:31 glb => bool; 888 } 889 890 /// Mask for job interrupts. 891 /// 892 /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it. 893 pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 { 894 /// Enable CSG request interrupts. 895 30:0 csg; 896 /// Enable GLB request interrupt. 897 31:31 glb => bool; 898 } 899 900 /// Active job interrupts. Read only. 901 /// 902 /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and 903 /// [`JOB_IRQ_MASK`]. 904 pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c { 905 /// CSG request interrupt status. 906 30:0 csg; 907 /// GLB request interrupt status. 908 31:31 glb => bool; 909 } 910 } 911 } 912 913 /// These registers correspond to the MMU_CONTROL register page. 914 /// They are involved in MMU configuration and control. 915 pub(crate) mod mmu_control { 916 use kernel::register; 917 918 register! { 919 /// IRQ sources raw status. 920 /// 921 /// This register contains the raw unmasked interrupt sources for MMU status and exception 922 /// handling. 923 /// 924 /// Writing to this register forces bits on. 925 /// Use [`IRQ_CLEAR`] to clear interrupts. 926 pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 { 927 /// Page fault for address spaces. 928 15:0 page_fault; 929 /// Command completed in address spaces. 930 31:16 command_completed; 931 } 932 933 /// IRQ sources to clear. 934 /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`]. 935 pub(crate) IRQ_CLEAR(u32) @ 0x2004 { 936 /// Clear the PAGE_FAULT interrupt. 937 15:0 page_fault; 938 /// Clear the COMMAND_COMPLETED interrupt. 939 31:16 command_completed; 940 } 941 942 /// IRQ sources enabled. 943 /// 944 /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it. 945 pub(crate) IRQ_MASK(u32) @ 0x2008 { 946 /// Enable the PAGE_FAULT interrupt. 947 15:0 page_fault; 948 /// Enable the COMMAND_COMPLETED interrupt. 949 31:16 command_completed; 950 } 951 952 /// IRQ status for enabled sources. Read only. 953 /// 954 /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`]. 955 pub(crate) IRQ_STATUS(u32) @ 0x200c { 956 /// PAGE_FAULT interrupt status. 957 15:0 page_fault; 958 /// COMMAND_COMPLETED interrupt status. 959 31:16 command_completed; 960 } 961 } 962 963 /// Per-address space registers ASn [0..15] within the MMU_CONTROL page. 964 /// 965 /// This array contains 16 instances of the MMU_AS_CONTROL register page. 966 pub(crate) mod mmu_as_control { 967 use core::convert::TryFrom; 968 969 use kernel::{ 970 error::{ 971 code::EINVAL, 972 Error, // 973 }, 974 num::Bounded, 975 register, // 976 }; 977 978 /// Maximum number of hardware address space slots. 979 /// The actual number of slots available is usually lower. 980 pub(crate) const MAX_AS: usize = 16; 981 982 /// Address space register stride. The elements in the array are spaced 64B apart. 983 const STRIDE: usize = 0x40; 984 985 register! { 986 /// Translation table base address. A 64-bit pointer. 987 /// 988 /// This field contains the address of the top level of a translation table structure. 989 /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero. 990 pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 { 991 /// Base address of the translation table. 992 63:0 base; 993 } 994 995 // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two 996 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 997 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 998 pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 { 999 31:0 value; 1000 } 1001 1002 pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 { 1003 31:0 value; 1004 } 1005 } 1006 1007 /// Helpers for MEMATTR Register. 1008 1009 #[derive(Copy, Clone, Debug, PartialEq)] 1010 #[repr(u8)] 1011 pub(crate) enum AllocPolicySelect { 1012 /// Ignore ALLOC_R/ALLOC_W fields. 1013 Impl = 2, 1014 /// Use ALLOC_R/ALLOC_W fields for allocation policy. 1015 Alloc = 3, 1016 } 1017 1018 impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect { 1019 type Error = Error; 1020 1021 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1022 match val.get() { 1023 2 => Ok(Self::Impl), 1024 3 => Ok(Self::Alloc), 1025 _ => Err(EINVAL), 1026 } 1027 } 1028 } 1029 1030 impl From<AllocPolicySelect> for Bounded<u8, 2> { 1031 fn from(val: AllocPolicySelect) -> Self { 1032 Bounded::try_new(val as u8).unwrap() 1033 } 1034 } 1035 1036 /// Coherency policy for memory attributes. Indicates the shareability of cached accesses. 1037 /// 1038 /// The hardware spec defines different interpretations of these values depending on 1039 /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation 1040 /// tables (all input addresses map to the same output address); it is deprecated and not 1041 /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY. 1042 #[derive(Copy, Clone, Debug, PartialEq)] 1043 #[repr(u8)] 1044 pub(crate) enum Coherency { 1045 /// Midgard inner domain coherency. 1046 /// 1047 /// Most flexible mode - can map non-coherent, internally coherent, and system/IO 1048 /// coherent memory. Used for non-cacheable memory in MAIR conversion. 1049 MidgardInnerDomain = 0, 1050 /// CPU inner domain coherency. 1051 /// 1052 /// Can map non-coherent and system/IO coherent memory. Used for write-back 1053 /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency. 1054 CpuInnerDomain = 1, 1055 /// CPU inner domain with shader coherency. 1056 /// 1057 /// Can map internally coherent and system/IO coherent memory. Used for 1058 /// GPU-internal shared buffers requiring shader coherency. 1059 CpuInnerDomainShaderCoh = 2, 1060 } 1061 1062 impl TryFrom<Bounded<u8, 2>> for Coherency { 1063 type Error = Error; 1064 1065 fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> { 1066 match val.get() { 1067 0 => Ok(Self::MidgardInnerDomain), 1068 1 => Ok(Self::CpuInnerDomain), 1069 2 => Ok(Self::CpuInnerDomainShaderCoh), 1070 _ => Err(EINVAL), 1071 } 1072 } 1073 } 1074 1075 impl From<Coherency> for Bounded<u8, 2> { 1076 fn from(val: Coherency) -> Self { 1077 Bounded::try_new(val as u8).unwrap() 1078 } 1079 } 1080 1081 #[derive(Copy, Clone, Debug, PartialEq)] 1082 #[repr(u8)] 1083 pub(crate) enum MemoryType { 1084 /// Normal memory (shared). 1085 Shared = 0, 1086 /// Normal memory, inner/outer non-cacheable. 1087 NonCacheable = 1, 1088 /// Normal memory, inner/outer write-back cacheable. 1089 WriteBack = 2, 1090 /// Triggers MEMORY_ATTRIBUTE_FAULT. 1091 Fault = 3, 1092 } 1093 1094 impl From<Bounded<u8, 2>> for MemoryType { 1095 fn from(val: Bounded<u8, 2>) -> Self { 1096 match val.get() { 1097 0 => Self::Shared, 1098 1 => Self::NonCacheable, 1099 2 => Self::WriteBack, 1100 3 => Self::Fault, 1101 _ => unreachable!(), 1102 } 1103 } 1104 } 1105 1106 impl From<MemoryType> for Bounded<u8, 2> { 1107 fn from(val: MemoryType) -> Self { 1108 Bounded::try_new(val as u8).unwrap() 1109 } 1110 } 1111 1112 register! { 1113 /// Stage 1 memory attributes (8-bit bitfield). 1114 /// 1115 /// This is not an actual register, but a bitfield definition used by the MEMATTR 1116 /// register. Each of the 8 bytes in MEMATTR follows this layout. 1117 MMU_MEMATTR_STAGE1(u8) @ 0x0 { 1118 /// Inner cache write allocation policy. 1119 0:0 alloc_w => bool; 1120 /// Inner cache read allocation policy. 1121 1:1 alloc_r => bool; 1122 /// Inner allocation policy select. 1123 3:2 alloc_sel ?=> AllocPolicySelect; 1124 /// Coherency policy. 1125 5:4 coherency ?=> Coherency; 1126 /// Memory type. 1127 7:6 memory_type => MemoryType; 1128 } 1129 } 1130 1131 impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 { 1132 type Error = Error; 1133 1134 fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> { 1135 Ok(Self::from_raw(val.get() as u8)) 1136 } 1137 } 1138 1139 impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> { 1140 fn from(val: MMU_MEMATTR_STAGE1) -> Self { 1141 Bounded::try_new(u64::from(val.into_raw())).unwrap() 1142 } 1143 } 1144 1145 register! { 1146 /// Memory attributes. 1147 /// 1148 /// Each address space can configure up to 8 different memory attribute profiles. 1149 /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout. 1150 pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 { 1151 7:0 attribute0 ?=> MMU_MEMATTR_STAGE1; 1152 15:8 attribute1 ?=> MMU_MEMATTR_STAGE1; 1153 23:16 attribute2 ?=> MMU_MEMATTR_STAGE1; 1154 31:24 attribute3 ?=> MMU_MEMATTR_STAGE1; 1155 39:32 attribute4 ?=> MMU_MEMATTR_STAGE1; 1156 47:40 attribute5 ?=> MMU_MEMATTR_STAGE1; 1157 55:48 attribute6 ?=> MMU_MEMATTR_STAGE1; 1158 63:56 attribute7 ?=> MMU_MEMATTR_STAGE1; 1159 } 1160 1161 // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two 1162 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1163 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1164 pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 { 1165 31:0 value; 1166 } 1167 1168 pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c { 1169 31:0 value; 1170 } 1171 1172 /// Lock region address for each address space. 1173 pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { 1174 /// Lock region size. 1175 5:0 size; 1176 /// Lock region base address. 1177 63:12 base; 1178 } 1179 1180 // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two 1181 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1182 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1183 pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 { 1184 31:0 value; 1185 } 1186 1187 pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 { 1188 31:0 value; 1189 } 1190 } 1191 1192 /// Helpers for MMU COMMAND register. 1193 #[derive(Copy, Clone, Debug, PartialEq)] 1194 #[repr(u8)] 1195 pub(crate) enum MmuCommand { 1196 /// No operation, nothing happens. 1197 Nop = 0, 1198 /// Propagate settings to the MMU. 1199 Update = 1, 1200 /// Lock an address region. 1201 Lock = 2, 1202 /// Unlock an address region. 1203 Unlock = 3, 1204 /// Clean and invalidate the L2 cache, then unlock. 1205 FlushPt = 4, 1206 /// Clean and invalidate all caches, then unlock. 1207 FlushMem = 5, 1208 } 1209 1210 impl TryFrom<Bounded<u32, 8>> for MmuCommand { 1211 type Error = Error; 1212 1213 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1214 match val.get() { 1215 0 => Ok(MmuCommand::Nop), 1216 1 => Ok(MmuCommand::Update), 1217 2 => Ok(MmuCommand::Lock), 1218 3 => Ok(MmuCommand::Unlock), 1219 4 => Ok(MmuCommand::FlushPt), 1220 5 => Ok(MmuCommand::FlushMem), 1221 _ => Err(EINVAL), 1222 } 1223 } 1224 } 1225 1226 impl From<MmuCommand> for Bounded<u32, 8> { 1227 fn from(cmd: MmuCommand) -> Self { 1228 (cmd as u8).into() 1229 } 1230 } 1231 1232 register! { 1233 /// MMU command register for each address space. Write only. 1234 pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 1235 7:0 command ?=> MmuCommand; 1236 } 1237 } 1238 1239 /// MMU exception types for FAULTSTATUS register. 1240 #[derive(Copy, Clone, Debug, PartialEq)] 1241 #[repr(u8)] 1242 pub(crate) enum MmuExceptionType { 1243 /// No error. 1244 Ok = 0x00, 1245 /// Invalid translation table entry, level 0. 1246 TranslationFault0 = 0xC0, 1247 /// Invalid translation table entry, level 1. 1248 TranslationFault1 = 0xC1, 1249 /// Invalid translation table entry, level 2. 1250 TranslationFault2 = 0xC2, 1251 /// Invalid translation table entry, level 3. 1252 TranslationFault3 = 0xC3, 1253 /// Invalid block descriptor. 1254 TranslationFault4 = 0xC4, 1255 /// Page permission error, level 0. 1256 PermissionFault0 = 0xC8, 1257 /// Page permission error, level 1. 1258 PermissionFault1 = 0xC9, 1259 /// Page permission error, level 2. 1260 PermissionFault2 = 0xCA, 1261 /// Page permission error, level 3. 1262 PermissionFault3 = 0xCB, 1263 /// Access flag not set, level 1. 1264 AccessFlag1 = 0xD9, 1265 /// Access flag not set, level 2. 1266 AccessFlag2 = 0xDA, 1267 /// Access flag not set, level 3. 1268 AccessFlag3 = 0xDB, 1269 /// Virtual address out of range. 1270 AddressSizeFaultIn = 0xE0, 1271 /// Physical address out of range, level 0. 1272 AddressSizeFaultOut0 = 0xE4, 1273 /// Physical address out of range, level 1. 1274 AddressSizeFaultOut1 = 0xE5, 1275 /// Physical address out of range, level 2. 1276 AddressSizeFaultOut2 = 0xE6, 1277 /// Physical address out of range, level 3. 1278 AddressSizeFaultOut3 = 0xE7, 1279 /// Page attribute error, level 0. 1280 MemoryAttributeFault0 = 0xE8, 1281 /// Page attribute error, level 1. 1282 MemoryAttributeFault1 = 0xE9, 1283 /// Page attribute error, level 2. 1284 MemoryAttributeFault2 = 0xEA, 1285 /// Page attribute error, level 3. 1286 MemoryAttributeFault3 = 0xEB, 1287 } 1288 1289 impl TryFrom<Bounded<u32, 8>> for MmuExceptionType { 1290 type Error = Error; 1291 1292 fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> { 1293 match val.get() { 1294 0x00 => Ok(MmuExceptionType::Ok), 1295 0xC0 => Ok(MmuExceptionType::TranslationFault0), 1296 0xC1 => Ok(MmuExceptionType::TranslationFault1), 1297 0xC2 => Ok(MmuExceptionType::TranslationFault2), 1298 0xC3 => Ok(MmuExceptionType::TranslationFault3), 1299 0xC4 => Ok(MmuExceptionType::TranslationFault4), 1300 0xC8 => Ok(MmuExceptionType::PermissionFault0), 1301 0xC9 => Ok(MmuExceptionType::PermissionFault1), 1302 0xCA => Ok(MmuExceptionType::PermissionFault2), 1303 0xCB => Ok(MmuExceptionType::PermissionFault3), 1304 0xD9 => Ok(MmuExceptionType::AccessFlag1), 1305 0xDA => Ok(MmuExceptionType::AccessFlag2), 1306 0xDB => Ok(MmuExceptionType::AccessFlag3), 1307 0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn), 1308 0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0), 1309 0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1), 1310 0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2), 1311 0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3), 1312 0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0), 1313 0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1), 1314 0xEA => Ok(MmuExceptionType::MemoryAttributeFault2), 1315 0xEB => Ok(MmuExceptionType::MemoryAttributeFault3), 1316 _ => Err(EINVAL), 1317 } 1318 } 1319 } 1320 1321 impl From<MmuExceptionType> for Bounded<u32, 8> { 1322 fn from(exc: MmuExceptionType) -> Self { 1323 (exc as u8).into() 1324 } 1325 } 1326 1327 /// Access type for MMU faults. 1328 #[derive(Copy, Clone, Debug, PartialEq)] 1329 #[repr(u8)] 1330 pub(crate) enum MmuAccessType { 1331 /// An atomic (read/write) transaction. 1332 Atomic = 0, 1333 /// An execute transaction. 1334 Execute = 1, 1335 /// A read transaction. 1336 Read = 2, 1337 /// A write transaction. 1338 Write = 3, 1339 } 1340 1341 impl From<Bounded<u32, 2>> for MmuAccessType { 1342 fn from(val: Bounded<u32, 2>) -> Self { 1343 match val.get() { 1344 0 => MmuAccessType::Atomic, 1345 1 => MmuAccessType::Execute, 1346 2 => MmuAccessType::Read, 1347 3 => MmuAccessType::Write, 1348 _ => unreachable!(), 1349 } 1350 } 1351 } 1352 1353 impl From<MmuAccessType> for Bounded<u32, 2> { 1354 fn from(access: MmuAccessType) -> Self { 1355 Bounded::try_new(access as u32).unwrap() 1356 } 1357 } 1358 1359 register! { 1360 /// Fault status register for each address space. Read only. 1361 pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { 1362 /// Exception type. 1363 7:0 exception_type ?=> MmuExceptionType; 1364 /// Access type. 1365 9:8 access_type => MmuAccessType; 1366 /// ID of the source that triggered the fault. 1367 31:16 source_id; 1368 } 1369 1370 /// Fault address for each address space. Read only. 1371 pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 { 1372 31:0 pointer; 1373 } 1374 1375 pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 { 1376 31:0 pointer; 1377 } 1378 1379 /// MMU status register for each address space. Read only. 1380 pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 { 1381 /// External address space command is active, a 1-bit boolean flag. 1382 0:0 active_ext => bool; 1383 /// Internal address space command is active, a 1-bit boolean flag. 1384 1:1 active_int => bool; 1385 } 1386 } 1387 1388 /// Helpers for TRANSCFG register. 1389 /// 1390 /// Address space mode for TRANSCFG register. 1391 #[derive(Copy, Clone, Debug, PartialEq)] 1392 #[repr(u8)] 1393 pub(crate) enum AddressSpaceMode { 1394 /// The MMU forces all memory access to fail with a decode fault. 1395 Unmapped = 1, 1396 /// All input addresses map to the same output address (deprecated). 1397 Identity = 2, 1398 /// Translation tables interpreted according to AArch64 4kB granule specification. 1399 Aarch64_4K = 6, 1400 /// Translation tables interpreted according to AArch64 64kB granule specification. 1401 Aarch64_64K = 8, 1402 } 1403 1404 impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode { 1405 type Error = Error; 1406 1407 fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> { 1408 match val.get() { 1409 1 => Ok(AddressSpaceMode::Unmapped), 1410 2 => Ok(AddressSpaceMode::Identity), 1411 6 => Ok(AddressSpaceMode::Aarch64_4K), 1412 8 => Ok(AddressSpaceMode::Aarch64_64K), 1413 _ => Err(EINVAL), 1414 } 1415 } 1416 } 1417 1418 impl From<AddressSpaceMode> for Bounded<u64, 4> { 1419 fn from(mode: AddressSpaceMode) -> Self { 1420 Bounded::try_new(mode as u64).unwrap() 1421 } 1422 } 1423 1424 /// Input address range restriction for TRANSCFG register. 1425 #[derive(Copy, Clone, Debug, PartialEq)] 1426 #[repr(u8)] 1427 pub(crate) enum InaBits { 1428 /// Invalid VA range (reset value). 1429 Reset = 0, 1430 /// 48-bit VA range. 1431 Bits48 = 7, 1432 /// 47-bit VA range. 1433 Bits47 = 8, 1434 /// 46-bit VA range. 1435 Bits46 = 9, 1436 /// 45-bit VA range. 1437 Bits45 = 10, 1438 /// 44-bit VA range. 1439 Bits44 = 11, 1440 /// 43-bit VA range. 1441 Bits43 = 12, 1442 /// 42-bit VA range. 1443 Bits42 = 13, 1444 /// 41-bit VA range. 1445 Bits41 = 14, 1446 /// 40-bit VA range. 1447 Bits40 = 15, 1448 /// 39-bit VA range. 1449 Bits39 = 16, 1450 /// 38-bit VA range. 1451 Bits38 = 17, 1452 /// 37-bit VA range. 1453 Bits37 = 18, 1454 /// 36-bit VA range. 1455 Bits36 = 19, 1456 /// 35-bit VA range. 1457 Bits35 = 20, 1458 /// 34-bit VA range. 1459 Bits34 = 21, 1460 /// 33-bit VA range. 1461 Bits33 = 22, 1462 /// 32-bit VA range. 1463 Bits32 = 23, 1464 /// 31-bit VA range. 1465 Bits31 = 24, 1466 /// 30-bit VA range. 1467 Bits30 = 25, 1468 /// 29-bit VA range. 1469 Bits29 = 26, 1470 /// 28-bit VA range. 1471 Bits28 = 27, 1472 /// 27-bit VA range. 1473 Bits27 = 28, 1474 /// 26-bit VA range. 1475 Bits26 = 29, 1476 /// 25-bit VA range. 1477 Bits25 = 30, 1478 } 1479 1480 impl TryFrom<Bounded<u64, 5>> for InaBits { 1481 type Error = Error; 1482 1483 fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> { 1484 match val.get() { 1485 0 => Ok(InaBits::Reset), 1486 7 => Ok(InaBits::Bits48), 1487 8 => Ok(InaBits::Bits47), 1488 9 => Ok(InaBits::Bits46), 1489 10 => Ok(InaBits::Bits45), 1490 11 => Ok(InaBits::Bits44), 1491 12 => Ok(InaBits::Bits43), 1492 13 => Ok(InaBits::Bits42), 1493 14 => Ok(InaBits::Bits41), 1494 15 => Ok(InaBits::Bits40), 1495 16 => Ok(InaBits::Bits39), 1496 17 => Ok(InaBits::Bits38), 1497 18 => Ok(InaBits::Bits37), 1498 19 => Ok(InaBits::Bits36), 1499 20 => Ok(InaBits::Bits35), 1500 21 => Ok(InaBits::Bits34), 1501 22 => Ok(InaBits::Bits33), 1502 23 => Ok(InaBits::Bits32), 1503 24 => Ok(InaBits::Bits31), 1504 25 => Ok(InaBits::Bits30), 1505 26 => Ok(InaBits::Bits29), 1506 27 => Ok(InaBits::Bits28), 1507 28 => Ok(InaBits::Bits27), 1508 29 => Ok(InaBits::Bits26), 1509 30 => Ok(InaBits::Bits25), 1510 _ => Err(EINVAL), 1511 } 1512 } 1513 } 1514 1515 impl From<InaBits> for Bounded<u64, 5> { 1516 fn from(bits: InaBits) -> Self { 1517 Bounded::try_new(bits as u64).unwrap() 1518 } 1519 } 1520 1521 /// Translation table memory attributes for TRANSCFG register. 1522 #[derive(Copy, Clone, Debug, PartialEq)] 1523 #[repr(u8)] 1524 pub(crate) enum PtwMemattr { 1525 /// Invalid (reset value, not valid for enabled address space). 1526 Invalid = 0, 1527 /// Normal memory, inner/outer non-cacheable. 1528 NonCacheable = 1, 1529 /// Normal memory, inner/outer write-back cacheable. 1530 WriteBack = 2, 1531 } 1532 1533 impl TryFrom<Bounded<u64, 2>> for PtwMemattr { 1534 type Error = Error; 1535 1536 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1537 match val.get() { 1538 0 => Ok(PtwMemattr::Invalid), 1539 1 => Ok(PtwMemattr::NonCacheable), 1540 2 => Ok(PtwMemattr::WriteBack), 1541 _ => Err(EINVAL), 1542 } 1543 } 1544 } 1545 1546 impl From<PtwMemattr> for Bounded<u64, 2> { 1547 fn from(attr: PtwMemattr) -> Self { 1548 Bounded::try_new(attr as u64).unwrap() 1549 } 1550 } 1551 1552 /// Translation table memory shareability for TRANSCFG register. 1553 #[derive(Copy, Clone, Debug, PartialEq)] 1554 #[repr(u8)] 1555 #[allow(clippy::enum_variant_names)] 1556 pub(crate) enum PtwShareability { 1557 /// Non-shareable. 1558 NonShareable = 0, 1559 /// Outer shareable. 1560 OuterShareable = 2, 1561 /// Inner shareable. 1562 InnerShareable = 3, 1563 } 1564 1565 impl TryFrom<Bounded<u64, 2>> for PtwShareability { 1566 type Error = Error; 1567 1568 fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> { 1569 match val.get() { 1570 0 => Ok(PtwShareability::NonShareable), 1571 2 => Ok(PtwShareability::OuterShareable), 1572 3 => Ok(PtwShareability::InnerShareable), 1573 _ => Err(EINVAL), 1574 } 1575 } 1576 } 1577 1578 impl From<PtwShareability> for Bounded<u64, 2> { 1579 fn from(sh: PtwShareability) -> Self { 1580 Bounded::try_new(sh as u64).unwrap() 1581 } 1582 } 1583 1584 register! { 1585 /// Translation configuration and control. 1586 pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { 1587 /// Address space mode. 1588 3:0 mode ?=> AddressSpaceMode; 1589 /// Address input restriction. 1590 10:6 ina_bits ?=> InaBits; 1591 /// Address output restriction. 1592 18:14 outa_bits; 1593 /// Translation table concatenation enable, a 1-bit boolean flag. 1594 22:22 sl_concat_en => bool; 1595 /// Translation table memory attributes. 1596 25:24 ptw_memattr ?=> PtwMemattr; 1597 /// Translation table memory shareability. 1598 29:28 ptw_sh ?=> PtwShareability; 1599 /// Inner read allocation hint for translation table walks, a 1-bit boolean flag. 1600 30:30 r_allocate => bool; 1601 /// Disable hierarchical access permissions. 1602 33:33 disable_hier_ap => bool; 1603 /// Disable access fault checking. 1604 34:34 disable_af_fault => bool; 1605 /// Disable execution on all writable pages. 1606 35:35 wxn => bool; 1607 /// Enable execution on readable pages. 1608 36:36 xreadable => bool; 1609 /// Page-based hardware attributes for translation table walks. 1610 63:60 ptw_pbha; 1611 } 1612 1613 // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two 1614 // 32-bit halves. Define it as separate low/high u32 registers so accesses match 1615 // the MMIO register layout and do not rely on native 64-bit MMIO transactions. 1616 pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 { 1617 31:0 value; 1618 } 1619 1620 pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 { 1621 31:0 value; 1622 } 1623 1624 /// Extra fault information for each address space. Read only. 1625 pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 { 1626 31:0 value; 1627 } 1628 1629 pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c { 1630 31:0 value; 1631 } 1632 } 1633 } 1634 } 1635 1636 /// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages. 1637 pub(crate) mod doorbell_block { 1638 use kernel::register; 1639 1640 /// Number of doorbells available. 1641 pub(crate) const NUM_DOORBELLS: usize = 64; 1642 1643 /// Doorbell block stride (64KiB). 1644 /// 1645 /// Each block occupies a full page, allowing it to be mapped 1646 /// separately into a virtual address space. 1647 const STRIDE: usize = 0x10000; 1648 1649 register! { 1650 /// Doorbell request register. Write-only. 1651 pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { 1652 /// Doorbell set. Writing 1 triggers the doorbell. 1653 0:0 ring => bool; 1654 } 1655 } 1656 } 1657