1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/elf_common.h> 30#include <sys/syscall.h> 31#include <machine/asm.h> 32#include <machine/armreg.h> 33#include <machine/cpu.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42/* 43 * Loads a 64-bit value into reg using 1 to 4 mov/movk instructions. 44 * This can be used early on when we don't know the CPUs endianness. 45 */ 46.macro mov_q reg, val 47 movz \reg, :abs_g0_nc:\val 48.if (\val >> 16) & 0xffff != 0 49 movk \reg, :abs_g1_nc:\val 50.endif 51.if (\val >> 32) & 0xffff != 0 52 movk \reg, :abs_g2_nc:\val 53.endif 54.if (\val >> 48) & 0xffff != 0 55 movk \reg, :abs_g3:\val 56.endif 57.endm 58 59#if PAGE_SIZE == PAGE_SIZE_16K 60/* 61 * The number of level 3 tables to create. 32 will allow for 1G of address 62 * space, the same as a single level 2 page with 4k pages. 63 */ 64#define L3_PAGE_COUNT 32 65#elif PAGE_SIZE == PAGE_SIZE_4K 66/* 67 * Space for a level 3 table holding the end of the executable memory and 68 * the start of the non-executable data. 69 */ 70#define L3_PAGE_COUNT 1 71#endif 72 73/* 74 * The size of our bootstrap stack. 75 */ 76#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 77 78 .globl kernbase 79 .set kernbase, KERNBASE 80 81/* 82 * We assume: 83 * MMU on with an identity map, or off 84 * D-Cache: off 85 * I-Cache: on or off 86 * We are loaded at a 2MiB aligned address 87 */ 88 89ENTRY(_start) 90 /* Enter the kernel exception level */ 91 bl enter_kernel_el 92 93 /* Set the context id */ 94 msr contextidr_el1, xzr 95 96 /* Get the virt -> phys offset */ 97 bl get_load_phys_addr 98 99 /* 100 * At this point: 101 * x28 = Our physical load address 102 */ 103 104 /* Create the page tables */ 105 bl create_pagetables 106 107 /* 108 * At this point: 109 * x27 = TTBR0 table 110 * x24 = TTBR1 table 111 * x22 = PTE shareability attributes 112 * x21 = BTI guarded page attribute if supported 113 */ 114 115 /* Enable the mmu */ 116 bl start_mmu 117 118 /* Load the new ttbr0 pagetable */ 119 adrp x27, pagetable_l0_ttbr0 120 add x27, x27, :lo12:pagetable_l0_ttbr0 121 122 /* Jump to the virtual address space */ 123 ldr x15, .Lvirtdone 124 br x15 125 126virtdone: 127 BTI_J 128 129 /* Set up the stack */ 130 adrp x25, initstack_end 131 add sp, x25, :lo12:initstack_end 132 133 /* Zero the BSS */ 134 ldr x15, .Lbss 135 ldr x14, .Lend 1361: 137 stp xzr, xzr, [x15], #16 138 cmp x15, x14 139 b.lo 1b 140 141#if defined(PERTHREAD_SSP) 142 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 143 adrp x15, boot_canary 144 add x15, x15, :lo12:boot_canary 145 msr sp_el0, x15 146#endif 147 148 /* Backup the module pointer */ 149 mov x1, x0 150 151 sub sp, sp, #BOOTPARAMS_SIZE 152 mov x0, sp 153 154 str x1, [x0, #BP_MODULEP] 155 adrp x25, initstack 156 add x25, x25, :lo12:initstack 157 str x25, [x0, #BP_KERN_STACK] 158 str x27, [x0, #BP_KERN_TTBR0] 159 str w23, [x0, #BP_BOOT_EL] 160 161 /* Set these before they are used in kasan_init_early */ 162 adrp x1, pmap_sh_attr 163 str x22, [x1, :lo12:pmap_sh_attr] 164#ifdef __ARM_FEATURE_BTI_DEFAULT 165 adrp x1, pmap_gp_attr 166 str x21, [x1, :lo12:pmap_gp_attr] 167#endif 168 169#ifdef KASAN 170 /* Save bootparams */ 171 mov x19, x0 172 173 /* Bootstrap an early shadow map for the boot stack. */ 174 ldr x0, [x0, #BP_KERN_STACK] 175 ldr x1, =BOOT_STACK_SIZE 176 bl kasan_init_early 177 178 /* Restore bootparams */ 179 mov x0, x19 180#endif 181 182 /* trace back starts here */ 183 mov fp, #0 184 /* Branch to C code */ 185 bl initarm 186 /* We are done with the boot params */ 187 add sp, sp, #BOOTPARAMS_SIZE 188 189 /* 190 * Enable pointer authentication in the kernel. We set the keys for 191 * thread0 in initarm so have to wait until it returns to enable it. 192 * If we were to enable it in initarm then any authentication when 193 * returning would fail as it was called with pointer authentication 194 * disabled. 195 */ 196 bl ptrauth_start 197 198 bl mi_startup 199 200 /* We should not get here */ 201 brk 0 202 203 .align 3 204.Lvirtdone: 205 .quad virtdone 206.Lbss: 207 .quad __bss_start 208.Lend: 209 .quad __bss_end 210END(_start) 211 212#ifdef SMP 213/* 214 * void 215 * mpentry_psci(unsigned long) 216 * 217 * Called by a core when it is being brought online with psci. 218 * The data in x0 is passed straight to init_secondary. 219 */ 220ENTRY(mpentry_psci) 221 mov x26, xzr 222 b mpentry_common 223END(mpentry_psci) 224 225/* 226 * void 227 * mpentry_spintable(void) 228 * 229 * Called by a core when it is being brought online with a spin-table. 230 * Reads the new CPU ID and passes this to init_secondary. 231 */ 232ENTRY(mpentry_spintable) 233 ldr x26, =spintable_wait 234 b mpentry_common 235END(mpentry_spintable) 236 237/* Wait for the current CPU to be released */ 238LENTRY(spintable_wait) 239 /* Read the affinity bits from mpidr_el1 */ 240 mrs x1, mpidr_el1 241 ldr x2, =CPU_AFF_MASK 242 and x1, x1, x2 243 244 adrp x2, ap_cpuid 2451: 246 ldr x0, [x2, :lo12:ap_cpuid] 247 cmp x0, x1 248 b.ne 1b 249 250 str xzr, [x2, :lo12:ap_cpuid] 251 dsb sy 252 sev 253 254 ret 255LEND(mpentry_spintable) 256 257LENTRY(mpentry_common) 258 /* Disable interrupts */ 259 msr daifset, #DAIF_INTR 260 261 /* Enter the kernel exception level */ 262 bl enter_kernel_el 263 264 /* Set the context id */ 265 msr contextidr_el1, xzr 266 267 /* Load the kernel page table */ 268 adrp x24, pagetable_l0_ttbr1 269 add x24, x24, :lo12:pagetable_l0_ttbr1 270 /* Load the identity page table */ 271 adrp x27, pagetable_l0_ttbr0_bootstrap 272 add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 273 274 /* Enable the mmu */ 275 bl start_mmu 276 277 /* Load the new ttbr0 pagetable */ 278 adrp x27, pagetable_l0_ttbr0 279 add x27, x27, :lo12:pagetable_l0_ttbr0 280 281 /* Jump to the virtual address space */ 282 ldr x15, =mp_virtdone 283 br x15 284 285mp_virtdone: 286 BTI_J 287 288 /* 289 * Allow this CPU to wait until the kernel is ready for it, 290 * e.g. with spin-table but each CPU uses the same release address 291 */ 292 cbz x26, 1f 293 blr x26 2941: 295 296 /* Start using the AP boot stack */ 297 adrp x4, bootstack 298 ldr x4, [x4, :lo12:bootstack] 299 mov sp, x4 300 301#if defined(PERTHREAD_SSP) 302 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 303 adrp x15, boot_canary 304 add x15, x15, :lo12:boot_canary 305 msr sp_el0, x15 306#endif 307 308 /* Load the kernel ttbr0 pagetable */ 309 msr ttbr0_el1, x27 310 isb 311 312 /* Invalidate the TLB */ 313 tlbi vmalle1 314 dsb sy 315 isb 316 317 /* 318 * Initialize the per-CPU pointer before calling into C code, for the 319 * benefit of kernel sanitizers. 320 */ 321 adrp x18, bootpcpu 322 ldr x18, [x18, :lo12:bootpcpu] 323 msr tpidr_el1, x18 324 325 b init_secondary 326LEND(mpentry_common) 327 328ENTRY(mp_cpu_spinloop) 3290: 330 wfe 331 ldr x0, mp_cpu_spin_table_release_addr 332 cbz x0, 0b 333 blr x0 334 .globl mp_cpu_spin_table_release_addr 335mp_cpu_spin_table_release_addr: 336 .quad 0 337 .globl mp_cpu_spinloop_end 338mp_cpu_spinloop_end: 339END(mp_cpu_spinloop) 340#endif 341 342/* 343 * Enter the exception level the kernel will use: 344 * 345 * - If in EL1 continue in EL1 346 * - If the CPU supports FEAT_VHE then set HCR_E2H and HCR_TGE and continue 347 * in EL2 348 * - Configure EL2 to support running the kernel at EL1 and exit to that 349 */ 350LENTRY(enter_kernel_el) 351 mrs x23, CurrentEL 352 and x23, x23, #(CURRENTEL_EL_MASK) 353 cmp x23, #(CURRENTEL_EL_EL2) 354 b.eq 1f 355 356 /* 357 * Ensure there are no memory operations here. If the boot loader 358 * enters the kernel in big-endian mode then loading sctlr will 359 * be incorrect. As instructions are the same in both endians it is 360 * safe to use mov instructions. 361 */ 362 mov_q x2, SCTLR_MMU_OFF 363 msr sctlr_el1, x2 364 /* 365 * SCTLR_EOS is set to make eret a context synchronizing event. We 366 * need an isb here to ensure it's observed by later instructions, 367 * but don't need it in the eret below. 368 */ 369 isb 370 371 /* 372 * Ensure SPSR_EL1 and pstate are in sync. The only way to set the 373 * latter is to set the former and return from an exception with eret. 374 */ 375 mov x2, #(PSR_DAIF | PSR_M_EL1h) 376 msr spsr_el1, x2 377 msr elr_el1, lr 378 eret 379 3801: 381 dsb sy 382 /* 383 * Set just the reserved bits in sctlr_el2. This will disable the 384 * MMU which may have broken the kernel if we enter the kernel in 385 * EL2, e.g. when using VHE. 386 * 387 * As with sctlr_el1 above use mov instructions to ensure there are 388 * no memory operations. 389 */ 390 mov_q x2, (SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) 391 msr sctlr_el2, x2 392 isb 393 394 /* 395 * The hardware is now in little-endian mode so memory operations 396 * are safe. 397 */ 398 399 /* Configure the Hypervisor */ 400 ldr x2, =(HCR_RW | HCR_APK | HCR_API | HCR_ATA) 401 msr hcr_el2, x2 402 403 /* Stash value of HCR_EL2 for later */ 404 isb 405 mrs x4, hcr_el2 406 407 /* Load the Virtualization Process ID Register */ 408 mrs x2, midr_el1 409 msr vpidr_el2, x2 410 411 /* Load the Virtualization Multiprocess ID Register */ 412 mrs x2, mpidr_el1 413 msr vmpidr_el2, x2 414 415 /* Set the initial sctlr_el1 */ 416 ldr x2, =SCTLR_MMU_OFF 417 msr sctlr_el1, x2 418 419 /* Check for VHE */ 420 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, VH, IMPL, .Lno_vhe) 421 422 /* 423 * The kernel will be running in EL2, route exceptions here rather 424 * than EL1. 425 */ 426 orr x4, x4, #HCR_E2H 427 orr x4, x4, #HCR_TGE 428 msr hcr_el2, x4 429 isb 430 431 msr SCTLR_EL12_REG, x2 432 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ 433 ldr x3, =(CNTHCTL_E2H_EL1PCTEN_NOTRAP | CNTHCTL_E2H_EL1PTEN_NOTRAP) 434 ldr x5, =(PSR_DAIF | PSR_M_EL2h) 435 b .Ldone_vhe 436 437.Lno_vhe: 438 /* Hypervisor trap functions */ 439 adrp x2, hyp_stub_vectors 440 add x2, x2, :lo12:hyp_stub_vectors 441 msr vbar_el2, x2 442 443 ldr x2, =(CPTR_RES1) 444 ldr x3, =(CNTHCTL_EL1PCTEN_NOTRAP | CNTHCTL_EL1PCEN_NOTRAP) 445 ldr x5, =(PSR_DAIF | PSR_M_EL1h) 446 447 /* Enable SPE at EL1 via Monitor Debug Configuration Register */ 448 mov x6, MDCR_EL2_E2PB_EL1_0_NO_TRAP 449 msr mdcr_el2, x6 450 451.Ldone_vhe: 452 453 msr cptr_el2, x2 454 /* Enable access to the physical timers at EL1 */ 455 msr cnthctl_el2, x3 456 /* Set the return PSTATE */ 457 msr spsr_el2, x5 458 459 /* 460 * Configure the Extended Hypervisor register. This is only valid if 461 * FEAT_HCX is enabled. 462 */ 463 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, HCX, IMPL, 2f) 464 /* Extended Hypervisor Configuration */ 465 msr HCRX_EL2_REG, xzr 466 isb 4672: 468 469 /* Don't trap to EL2 for CP15 traps */ 470 msr hstr_el2, xzr 471 472 /* Set the counter offset to a known value */ 473 msr cntvoff_el2, xzr 474 475 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 476 msr vttbr_el2, xzr 477 478 /* Check the CPU supports GIC, and configure the CPU interface */ 479 CHECK_CPU_FEAT(x2, ID_AA64PFR0, GIC, CPUIF_EN, 3f) 480 481 mrs x2, icc_sre_el2 482 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 483 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 484 msr icc_sre_el2, x2 4853: 486 487 /* Set the address to return to our return address */ 488 msr elr_el2, x30 489 isb 490 491 eret 492LEND(enter_kernel_el) 493 494/* Turn off the MMU. Install ttbr0 from the bootstrap page table, and go there. 495 * Does not return. 496 * - x0 - target address to jump to after stopping the MMU. 497 * - x1 - kernel load address 498 */ 499ENTRY(stop_mmu) 500 mov x16, x0 /* Save target. */ 501 ldr x2, =(1f - KERNBASE) 502 add x17, x1, x2 503 ldr x3, =(pagetable_l0_ttbr0_bootstrap - KERNBASE) 504 add x1, x1, x3 505 msr ttbr0_el1, x1 506 isb 507 br x17 5081: 509 BTI_J 510 mrs x0, sctlr_el1 511 bic x0, x0, SCTLR_M 512 bic x0, x0, SCTLR_C 513 msr sctlr_el1, x0 514 isb 515 br x16 516END(stop_mmu) 517/* 518 * Get the physical address the kernel was loaded at. 519 */ 520LENTRY(get_load_phys_addr) 521 /* Load the offset of get_load_phys_addr from KERNBASE */ 522 ldr x28, =(get_load_phys_addr - KERNBASE) 523 /* Load the physical address of get_load_phys_addr */ 524 adr x29, get_load_phys_addr 525 /* Find the physical address of KERNBASE, i.e. our load address */ 526 sub x28, x29, x28 527 ret 528LEND(get_load_phys_addr) 529 530/* 531 * This builds the page tables containing the identity map, and the kernel 532 * virtual map. 533 * 534 * It relys on: 535 * We were loaded to an address that is on a 2MiB boundary 536 * All the memory must not cross a 1GiB boundaty 537 * x28 contains the physical address we were loaded from 538 * 539 * There are 7 or 8 pages before that address for the page tables 540 * The pages used are: 541 * - The Kernel L3 tables (only for 16k kernel) 542 * - The Kernel L2 table 543 * - The Kernel L1 table 544 * - The Kernel L0 table (TTBR1) 545 * - The identity (PA = VA) L2 table 546 * - The identity (PA = VA) L1 table 547 * - The identity (PA = VA) L0 table (Early TTBR0) 548 * - The Kernel empty L0 table (Late TTBR0) 549 */ 550LENTRY(create_pagetables) 551 /* Save the Link register */ 552 mov x5, x30 553 554 /* Clean the page table */ 555 adrp x6, pagetable 556 add x6, x6, :lo12:pagetable 557 adrp x27, pagetable_end 558 add x27, x27, :lo12:pagetable_end 5591: 560 stp xzr, xzr, [x6], #16 561 stp xzr, xzr, [x6], #16 562 stp xzr, xzr, [x6], #16 563 stp xzr, xzr, [x6], #16 564 cmp x6, x27 565 b.lo 1b 566 567#ifdef __ARM_FEATURE_BTI_DEFAULT 568 /* 569 * Check if the CPU supports BTI 570 */ 571 mrs x6, id_aa64pfr1_el1 /* Read the ID register */ 572 and x6, x6, ID_AA64PFR1_BT_MASK /* Mask the field we need */ 573 cmp x6, xzr /* Check it's non-zero */ 574 cset x6, ne /* x6 is set if non-zero */ 575 lsl x21, x6, ATTR_S1_GP_SHIFT /* Shift to the correct bit */ 576#endif 577 578 /* 579 * Find the shareability attribute we should use. If FEAT_LPA2 is 580 * enabled then the shareability field is moved from the page table 581 * to tcr_el1 and the bits in the page table are reused by the 582 * address field. 583 */ 584#if PAGE_SIZE == PAGE_SIZE_4K 585#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK 586#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 587#elif PAGE_SIZE == PAGE_SIZE_16K 588#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK 589#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 590#else 591#error Unsupported page size 592#endif 593 mrs x6, id_aa64mmfr0_el1 594 mov x7, LPA2_VAL 595 and x6, x6, LPA2_MASK 596 cmp x6, x7 597 ldr x22, =(ATTR_SH(ATTR_SH_IS)) 598 csel x22, xzr, x22, eq 599#undef LPA2_MASK 600#undef LPA2_VAL 601 602 /* 603 * Build the TTBR1 maps. 604 */ 605 606 /* Find the size of the kernel */ 607 mov x6, #(KERNBASE) 608 609#if defined(LINUX_BOOT_ABI) 610 /* X19 is used as 'map FDT data' flag */ 611 mov x19, xzr 612 613 /* No modules or FDT pointer ? */ 614 cbz x0, booti_no_fdt 615 616 /* 617 * Test if x0 points to modules descriptor(virtual address) or 618 * to FDT (physical address) 619 */ 620 cmp x0, x6 /* x6 is #(KERNBASE) */ 621 b.lo booti_fdt 622#endif 623 624 /* Booted with modules pointer */ 625 /* Find modulep - begin */ 626 sub x8, x0, x6 627 /* 628 * Add space for the module data. When PAGE_SIZE is 4k this will 629 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 630 * larger it will be at least as large as we use smaller level 3 631 * pages. 632 */ 633 ldr x7, =((6 * 1024 * 1024) - 1) 634 add x8, x8, x7 635 b common 636 637#if defined(LINUX_BOOT_ABI) 638booti_fdt: 639 /* Booted by U-Boot booti with FDT data */ 640 /* Set 'map FDT data' flag */ 641 mov x19, #1 642 643booti_no_fdt: 644 /* Booted by U-Boot booti without FTD data */ 645 /* Find the end - begin */ 646 ldr x7, .Lend 647 sub x8, x7, x6 648 649 /* 650 * Add one 2MiB page for copy of FDT data (maximum FDT size), 651 * one for metadata and round up 652 */ 653 ldr x7, =(3 * L2_SIZE - 1) 654 add x8, x8, x7 655#endif 656 657common: 658#if PAGE_SIZE != PAGE_SIZE_4K 659 /* 660 * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned 661 * address, enabling the creation of L3C pages. However, when the page 662 * size is larger than 4k, L2 blocks are too large to map the kernel 663 * with 2M alignment. 664 */ 665#define PTE_SHIFT L3_SHIFT 666#define LL_PAGE_TABLE pagetable_l3_ttbr1 667#define BUILD_PTE_FUNC build_l3_page_pagetable 668#else 669#define PTE_SHIFT L2_SHIFT 670#define LL_PAGE_TABLE pagetable_l2_ttbr1 671#define BUILD_PTE_FUNC build_l2_block_pagetable 672#endif 673 674 /* Get the number of blocks/pages to allocate, rounded down */ 675 lsr x14, x8, #(PTE_SHIFT) 676 677 ldr x26, =etext 678#if PAGE_SIZE != PAGE_SIZE_4K 679 ldr x8, =((1 << PTE_SHIFT) - 1) 680 add x26, x26, x8 681#endif 682 mov x8, #(KERNBASE) 683 sub x25, x26, x8 684 lsr x25, x25, #(PTE_SHIFT) 685 686#if PAGE_SIZE == PAGE_SIZE_4K 687 /* Calculate the number of executable level 3 pages to create */ 688 lsr x26, x26, #(L3_SHIFT) 689 bfc x26, #(Ln_ENTRIES_SHIFT), #(64 - Ln_ENTRIES_SHIFT) 690 691 /* Build the L3 table holding the end of the exectuable code */ 692 lsl x15, x25, #(PTE_SHIFT) 693 adrp x6, pagetable_l3_ttbr1 694 add x6, x6, :lo12:pagetable_l3_ttbr1 695 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 696 ATTR_S1_AP(ATTR_S1_AP_RO)) 697 ldr x8, =(KERNBASE) 698 add x8, x8, x15 699 add x9, x28, x15 700 mov x10, x26 701 bl build_l3_page_pagetable 702 703 /* Build the remaining level 3 pages */ 704 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 705 lsl x27, x26, #(L3_SHIFT) 706 add x8, x8, x27 707 add x9, x28, x15 708 add x9, x9, x27 709 ldr x10, =(Ln_ENTRIES) 710 sub x10, x10, x26 711 bl build_l3_page_pagetable 712 713 /* Link the l2 -> l3 table */ 714 mov x9, x6 715 adrp x6, pagetable_l2_ttbr1 716 add x6, x6, :lo12:pagetable_l2_ttbr1 717 bl link_l2_pagetable 718#endif 719 720 /* Create the kernel space PTE table */ 721 adrp x6, LL_PAGE_TABLE 722 add x6, x6, :lo12:LL_PAGE_TABLE 723 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 724 ATTR_S1_AP(ATTR_S1_AP_RO)) 725 mov x8, #(KERNBASE) 726 mov x9, x28 727 mov x10, x25 728 bl BUILD_PTE_FUNC 729 730#if PAGE_SIZE == PAGE_SIZE_4K 731 /* Skip memory mapped through the L2 table */ 732 add x25, x25, #1 733#endif 734 735 /* Create the kernel space XN PTE table */ 736 lsl x10, x25, #(PTE_SHIFT) 737 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 738 ldr x8, =(KERNBASE) 739 add x8, x8, x10 740 add x9, x28, x10 741 sub x10, x14, x25 742 bl BUILD_PTE_FUNC 743 744#undef PTE_SHIFT 745#undef LL_PAGE_TABLE 746#undef BUILD_PTE_FUNC 747 748#if PAGE_SIZE != PAGE_SIZE_4K 749 /* Link the l2 -> l3 table */ 750 mov x9, x6 751 adrp x6, pagetable_l2_ttbr1 752 add x6, x6, :lo12:pagetable_l2_ttbr1 753 bl link_l2_pagetable 754#endif 755 756 /* Link the l1 -> l2 table */ 757 mov x9, x6 758 adrp x6, pagetable_l1_ttbr1 759 add x6, x6, :lo12:pagetable_l1_ttbr1 760 bl link_l1_pagetable 761 762 /* Link the l0 -> l1 table */ 763 mov x9, x6 764 adrp x6, pagetable_l0_ttbr1 765 add x6, x6, :lo12:pagetable_l0_ttbr1 766 mov x10, #1 767 bl link_l0_pagetable 768 769 /* Save the TTBR1 table physical address */ 770 mov x24, x6 771 772 /* 773 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 774 * They are only needed early on, so the VA = PA map is uncached. 775 */ 776 777 adrp x6, pagetable_l2_ttbr0_bootstrap 778 add x6, x6, :lo12:pagetable_l2_ttbr0_bootstrap 779 780 /* Create the VA = PA map */ 781 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 782 adrp x16, _start 783 and x16, x16, #(~L2_OFFSET) 784 mov x9, x16 /* PA start */ 785 mov x8, x16 /* VA start (== PA start) */ 786 mov x10, #1 787 bl build_l2_block_pagetable 788 789#if defined(SOCDEV_PA) 790 /* Create a table for the UART */ 791 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 792 ldr x9, =(L2_SIZE) 793 add x16, x16, x9 /* VA start */ 794 mov x8, x16 795 796 /* Store the socdev virtual address */ 797 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 798 adrp x9, socdev_va 799 str x17, [x9, :lo12:socdev_va] 800 801 ldr x9, =(SOCDEV_PA) /* PA start */ 802 mov x10, #1 803 bl build_l2_block_pagetable 804#endif 805 806#if defined(LINUX_BOOT_ABI) 807 /* Map FDT data ? */ 808 cbz x19, 1f 809 810 /* Create the mapping for FDT data (2 MiB max) */ 811 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 812 ldr x9, =(L2_SIZE) 813 add x16, x16, x9 /* VA start */ 814 mov x8, x16 815 mov x9, x0 /* PA start */ 816 /* Update the module pointer to point at the allocated memory */ 817 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 818 add x0, x0, x8 /* Add the aligned virtual address */ 819 820 mov x10, #1 821 bl build_l2_block_pagetable 822 8231: 824#endif 825 826 /* Link the l1 -> l2 table */ 827 mov x9, x6 828 adrp x6, pagetable_l1_ttbr0_bootstrap 829 add x6, x6, :lo12:pagetable_l1_ttbr0_bootstrap 830 bl link_l1_pagetable 831 832 /* Link the l0 -> l1 table */ 833 mov x9, x6 834 adrp x6, pagetable_l0_ttbr0_bootstrap 835 add x6, x6, :lo12:pagetable_l0_ttbr0_bootstrap 836 mov x10, #1 837 bl link_l0_pagetable 838 839 /* Save the TTBR0 table physical address */ 840 mov x27, x6 841 842 /* Restore the Link register */ 843 mov x30, x5 844 ret 845LEND(create_pagetables) 846 847/* 848 * Builds an L0 -> L1 table descriptor 849 * 850 * x6 = L0 table 851 * x8 = Virtual Address 852 * x9 = L1 PA (trashed) 853 * x10 = Entry count (trashed) 854 * x11, x12 and x13 are trashed 855 */ 856LENTRY(link_l0_pagetable) 857 /* 858 * Link an L0 -> L1 table entry. 859 */ 860 /* Find the table index */ 861 lsr x11, x8, #L0_SHIFT 862 and x11, x11, #L0_ADDR_MASK 863 864 /* Build the L0 block entry */ 865 mov x12, #L0_TABLE 866 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 867 868 /* Only use the output address bits */ 869 lsr x9, x9, #PAGE_SHIFT 8701: orr x13, x12, x9, lsl #PAGE_SHIFT 871 872 /* Store the entry */ 873 str x13, [x6, x11, lsl #3] 874 875 sub x10, x10, #1 876 add x11, x11, #1 877 add x9, x9, #1 878 cbnz x10, 1b 879 880 ret 881LEND(link_l0_pagetable) 882 883/* 884 * Builds an L1 -> L2 table descriptor 885 * 886 * x6 = L1 table 887 * x8 = Virtual Address 888 * x9 = L2 PA (trashed) 889 * x11, x12 and x13 are trashed 890 */ 891LENTRY(link_l1_pagetable) 892 /* 893 * Link an L1 -> L2 table entry. 894 */ 895 /* Find the table index */ 896 lsr x11, x8, #L1_SHIFT 897 and x11, x11, #Ln_ADDR_MASK 898 899 /* Build the L1 block entry */ 900 mov x12, #L1_TABLE 901 902 /* Only use the output address bits */ 903 lsr x9, x9, #PAGE_SHIFT 904 orr x13, x12, x9, lsl #PAGE_SHIFT 905 906 /* Store the entry */ 907 str x13, [x6, x11, lsl #3] 908 909 ret 910LEND(link_l1_pagetable) 911 912/* 913 * Builds count 2 MiB page table entry 914 * x6 = L2 table 915 * x7 = Block attributes 916 * x8 = VA start 917 * x9 = PA start (trashed) 918 * x10 = Entry count (trashed) 919 * x11, x12 and x13 are trashed 920 */ 921LENTRY(build_l2_block_pagetable) 922 /* 923 * Build the L2 table entry. 924 */ 925 /* Find the table index */ 926 lsr x11, x8, #L2_SHIFT 927 and x11, x11, #Ln_ADDR_MASK 928 929 /* Build the L2 block entry */ 930 orr x12, x7, #L2_BLOCK 931 orr x12, x12, #(ATTR_AF) 932 orr x12, x12, #(ATTR_S1_UXN) 933#ifdef __ARM_FEATURE_BTI_DEFAULT 934 orr x12, x12, x21 935#endif 936 /* Set the shareability attribute */ 937 orr x12, x12, x22 938 939 /* Only use the output address bits */ 940 lsr x9, x9, #L2_SHIFT 941 942 /* Set the physical address for this virtual address */ 9431: orr x13, x12, x9, lsl #L2_SHIFT 944 945 /* Store the entry */ 946 str x13, [x6, x11, lsl #3] 947 948 sub x10, x10, #1 949 add x11, x11, #1 950 add x9, x9, #1 951 cbnz x10, 1b 952 953 ret 954LEND(build_l2_block_pagetable) 955 956/* 957 * Builds an L2 -> L3 table descriptor 958 * 959 * x6 = L2 table 960 * x8 = Virtual Address 961 * x9 = L3 PA (trashed) 962 * x11, x12 and x13 are trashed 963 */ 964LENTRY(link_l2_pagetable) 965 /* 966 * Link an L2 -> L3 table entry. 967 */ 968 /* Find the table index */ 969 lsr x11, x8, #L2_SHIFT 970 and x11, x11, #Ln_ADDR_MASK 971 972 /* Build the L1 block entry */ 973 mov x12, #L2_TABLE 974 975 /* Only use the output address bits */ 976 lsr x9, x9, #PAGE_SHIFT 977 orr x13, x12, x9, lsl #PAGE_SHIFT 978 979 /* Store the entry */ 980 str x13, [x6, x11, lsl #3] 981 982 ret 983LEND(link_l2_pagetable) 984 985/* 986 * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create 987 * large page (L3C) mappings when the current VA and remaining count allow 988 * it. 989 * x6 = L3 table 990 * x7 = Block attributes 991 * x8 = VA start 992 * x9 = PA start (trashed) 993 * x10 = Entry count (trashed) 994 * x11, x12 and x13 are trashed 995 * 996 * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. 997 */ 998LENTRY(build_l3_page_pagetable) 999 cbz x10, 4f 1000 /* 1001 * Build the L3 table entry. 1002 */ 1003 /* Find the table index */ 1004 lsr x11, x8, #L3_SHIFT 1005 and x11, x11, #Ln_ADDR_MASK 1006 1007 /* Build the L3 page entry */ 1008 orr x12, x7, #L3_PAGE 1009 orr x12, x12, #(ATTR_AF) 1010 orr x12, x12, #(ATTR_S1_UXN) 1011#ifdef __ARM_FEATURE_BTI_DEFAULT 1012 orr x12, x12, x21 1013#endif 1014 /* Set the shareability attribute */ 1015 orr x12, x12, x22 1016 1017 /* Only use the output address bits */ 1018 lsr x9, x9, #L3_SHIFT 1019 1020 /* Check if an ATTR_CONTIGUOUS mapping is possible */ 10211: tst x11, #(L3C_ENTRIES - 1) 1022 b.ne 2f 1023 cmp x10, #L3C_ENTRIES 1024 b.lo 3f 1025 orr x12, x12, #(ATTR_CONTIGUOUS) 1026 b 2f 10273: and x12, x12, #(~ATTR_CONTIGUOUS) 1028 1029 /* Set the physical address for this virtual address */ 10302: orr x13, x12, x9, lsl #L3_SHIFT 1031 1032 /* Store the entry */ 1033 str x13, [x6, x11, lsl #3] 1034 1035 sub x10, x10, #1 1036 add x11, x11, #1 1037 add x9, x9, #1 1038 cbnz x10, 1b 10394: 1040 1041 ret 1042LEND(build_l3_page_pagetable) 1043 1044LENTRY(start_mmu) 1045 dsb sy 1046 1047 /* Load the exception vectors */ 1048 ldr x2, =exception_vectors 1049 msr vbar_el1, x2 1050 1051 /* Load ttbr0 and ttbr1 */ 1052 msr ttbr0_el1, x27 1053 msr ttbr1_el1, x24 1054 isb 1055 1056 /* Clear the Monitor Debug System control register */ 1057 msr mdscr_el1, xzr 1058 1059 /* Invalidate the TLB */ 1060 tlbi vmalle1is 1061 dsb ish 1062 isb 1063 1064 ldr x2, mair 1065 1066 /* 1067 * If FEAT_MTE2 is supported, configure GCR_EL1 and clear the TFSR 1068 * registers of any pending tag check faults 1069 */ 1070 CHECK_CPU_FEAT(x3, ID_AA64PFR1, MTE, MTE2, 1f) 1071 1072 /* Set GCR_EL1, non-zero tags excluded by default */ 1073 mov x3, #(GCR_Exclude_MASK | GCR_RRND) 1074 msr GCR_EL1_REG, x3 1075 1076 /* Clear any pending tag check faults */ 1077 msr TFSR_EL1_REG, xzr 1078 msr TFSRE0_EL1_REG, xzr 10791: 1080 msr mair_el1, x2 1081 1082 /* 1083 * Setup TCR according to the PARange and ASIDBits fields 1084 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 1085 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 1086 * to 1 only if the ASIDBits field equals 0b0010. 1087 */ 1088 ldr x2, tcr 1089 1090 /* If x22 contains a non-zero value then LPA2 is not implemented */ 1091 cbnz x22, .Lno_lpa2 1092 ldr x3, =(TCR_DS) 1093 orr x2, x2, x3 1094.Lno_lpa2: 1095 1096 mrs x3, id_aa64mmfr0_el1 1097 1098 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 1099 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 1100 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 1101 1102 /* Check if the HW supports 16 bit ASIDS */ 1103 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 1104 /* If so x3 == 1, else x3 == 0 */ 1105 cset x3, eq 1106 /* Set TCR.AS with x3 */ 1107 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 1108 1109 /* 1110 * Check if the HW supports access flag updates, and set 1111 * TCR_EL1.HA accordingly. The TCR_EL1.HD flag to enable 1112 * HW management of dirty state is set in C code as it may 1113 * need to be disabled because of CPU errata. 1114 */ 1115 CHECK_CPU_FEAT(x3, ID_AA64MMFR1, HAFDBS, AF, 1f) 1116 orr x2, x2, #(TCR_HA) 11171: 1118 1119 msr tcr_el1, x2 1120 1121 /* 1122 * Setup SCTLR. 1123 */ 1124 ldr x1, =SCTLR_MMU_ON 1125 msr sctlr_el1, x1 1126 isb 1127 1128 ret 1129 1130 .align 3 1131mair: 1132 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 1133 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 1134 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 1135 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 1136 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) | \ 1137 MAIR_ATTR(MAIR_NORMAL_TG, VM_MEMATTR_TAGGED) 1138tcr: 1139#if PAGE_SIZE == PAGE_SIZE_4K 1140#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 1141#elif PAGE_SIZE == PAGE_SIZE_16K 1142#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 1143#else 1144#error Unsupported page size 1145#endif 1146 1147 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 1148 TCR_SH1_IS | TCR_ORGN1_WBWA | TCR_IRGN1_WBWA | \ 1149 TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) 1150LEND(start_mmu) 1151 1152ENTRY(switch_stack) 1153 mov sp, x0 1154 mov x16, x1 1155 br x16 1156END(switch_stack) 1157 1158ENTRY(abort) 1159 b abort 1160END(abort) 1161 1162.bss 1163 .align PAGE_SHIFT 1164 .globl initstack_end 1165initstack: 1166 .space BOOT_STACK_SIZE 1167initstack_end: 1168 1169 .section .init_pagetable, "aw", %nobits 1170 .align PAGE_SHIFT 1171 /* 1172 * 6 initial tables (in the following order): 1173 * L2 for kernel (High addresses) 1174 * L1 for kernel 1175 * L0 for kernel 1176 * L1 bootstrap for user (Low addresses) 1177 * L0 bootstrap for user 1178 * L0 for user 1179 */ 1180 .globl pagetable_l0_ttbr1 1181 .globl pagetable_l0_ttbr0_bootstrap 1182pagetable: 1183pagetable_l3_ttbr1: 1184 .space (PAGE_SIZE * L3_PAGE_COUNT) 1185pagetable_l2_ttbr1: 1186 .space PAGE_SIZE 1187pagetable_l1_ttbr1: 1188 .space PAGE_SIZE 1189pagetable_l0_ttbr1: 1190 .space PAGE_SIZE 1191pagetable_l2_ttbr0_bootstrap: 1192 .space PAGE_SIZE 1193pagetable_l1_ttbr0_bootstrap: 1194 .space PAGE_SIZE 1195pagetable_l0_ttbr0_bootstrap: 1196 .space PAGE_SIZE 1197pagetable_l0_ttbr0: 1198 .space PAGE_SIZE 1199pagetable_end: 1200 1201el2_pagetable: 1202 .space PAGE_SIZE 1203 1204 .section .rodata, "a", %progbits 1205 .globl aarch32_sigcode 1206 .align 2 1207aarch32_sigcode: 1208 .word 0xe1a0000d // mov r0, sp 1209 .word 0xe2800040 // add r0, r0, #SIGF_UC 1210 .word 0xe59f700c // ldr r7, [pc, #12] 1211 .word 0xef000000 // swi #0 1212 .word 0xe59f7008 // ldr r7, [pc, #8] 1213 .word 0xef000000 // swi #0 1214 .word 0xeafffffa // b . - 16 1215 .word SYS_sigreturn 1216 .word SYS_exit 1217 .align 3 1218 .size aarch32_sigcode, . - aarch32_sigcode 1219aarch32_esigcode: 1220 .data 1221 .global sz_aarch32_sigcode 1222sz_aarch32_sigcode: 1223 .quad aarch32_esigcode - aarch32_sigcode 1224 1225GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 1226