1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/elf_common.h> 30#include <sys/syscall.h> 31#include <machine/asm.h> 32#include <machine/armreg.h> 33#include <machine/cpu.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42/* 43 * Loads a 64-bit value into reg using 1 to 4 mov/movk instructions. 44 * This can be used early on when we don't know the CPUs endianness. 45 */ 46.macro mov_q reg, val 47 mov \reg, :abs_g0_nc:\val 48.if (\val >> 16) & 0xffff != 0 49 movk \reg, :abs_g1_nc:\val 50.endif 51.if (\val >> 32) & 0xffff != 0 52 movk \reg, :abs_g2_nc:\val 53.endif 54.if (\val >> 48) & 0xffff != 0 55 movk \reg, :abs_g3:\val 56.endif 57.endm 58 59#if PAGE_SIZE == PAGE_SIZE_16K 60/* 61 * The number of level 3 tables to create. 32 will allow for 1G of address 62 * space, the same as a single level 2 page with 4k pages. 63 */ 64#define L3_PAGE_COUNT 32 65#elif PAGE_SIZE == PAGE_SIZE_4K 66/* 67 * Space for a level 3 table holding the end of the executable memory and 68 * the start of the non-executable data. 69 */ 70#define L3_PAGE_COUNT 1 71#endif 72 73/* 74 * The size of our bootstrap stack. 75 */ 76#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 77 78 .globl kernbase 79 .set kernbase, KERNBASE 80 81/* 82 * We assume: 83 * MMU on with an identity map, or off 84 * D-Cache: off 85 * I-Cache: on or off 86 * We are loaded at a 2MiB aligned address 87 */ 88 89ENTRY(_start) 90 /* Enter the kernel exception level */ 91 bl enter_kernel_el 92 93 /* Set the context id */ 94 msr contextidr_el1, xzr 95 96 /* Get the virt -> phys offset */ 97 bl get_load_phys_addr 98 99 /* 100 * At this point: 101 * x28 = Our physical load address 102 */ 103 104 /* Create the page tables */ 105 bl create_pagetables 106 107 /* 108 * At this point: 109 * x27 = TTBR0 table 110 * x24 = TTBR1 table 111 * x22 = PTE shareability attributes 112 * x21 = BTI guarded page attribute if supported 113 */ 114 115 /* Enable the mmu */ 116 bl start_mmu 117 118 /* Load the new ttbr0 pagetable */ 119 adrp x27, pagetable_l0_ttbr0 120 add x27, x27, :lo12:pagetable_l0_ttbr0 121 122 /* Jump to the virtual address space */ 123 ldr x15, .Lvirtdone 124 br x15 125 126virtdone: 127 BTI_J 128 129 /* Set up the stack */ 130 adrp x25, initstack_end 131 add x25, x25, :lo12:initstack_end 132 sub sp, x25, #PCB_SIZE 133 134 /* Zero the BSS */ 135 ldr x15, .Lbss 136 ldr x14, .Lend 1371: 138 stp xzr, xzr, [x15], #16 139 cmp x15, x14 140 b.lo 1b 141 142#if defined(PERTHREAD_SSP) 143 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 144 adrp x15, boot_canary 145 add x15, x15, :lo12:boot_canary 146 msr sp_el0, x15 147#endif 148 149 /* Backup the module pointer */ 150 mov x1, x0 151 152 sub sp, sp, #BOOTPARAMS_SIZE 153 mov x0, sp 154 155 str x1, [x0, #BP_MODULEP] 156 adrp x25, initstack 157 add x25, x25, :lo12:initstack 158 str x25, [x0, #BP_KERN_STACK] 159 str x27, [x0, #BP_KERN_TTBR0] 160 str x23, [x0, #BP_BOOT_EL] 161 162 /* Set these before they are used in kasan_init_early */ 163 adrp x1, pmap_sh_attr 164 str x22, [x1, :lo12:pmap_sh_attr] 165#ifdef __ARM_FEATURE_BTI_DEFAULT 166 adrp x1, pmap_gp_attr 167 str x21, [x1, :lo12:pmap_gp_attr] 168#endif 169 170#ifdef KASAN 171 /* Save bootparams */ 172 mov x19, x0 173 174 /* Bootstrap an early shadow map for the boot stack. */ 175 ldr x0, [x0, #BP_KERN_STACK] 176 ldr x1, =BOOT_STACK_SIZE 177 bl kasan_init_early 178 179 /* Restore bootparams */ 180 mov x0, x19 181#endif 182 183 /* trace back starts here */ 184 mov fp, #0 185 /* Branch to C code */ 186 bl initarm 187 /* We are done with the boot params */ 188 add sp, sp, #BOOTPARAMS_SIZE 189 190 /* 191 * Enable pointer authentication in the kernel. We set the keys for 192 * thread0 in initarm so have to wait until it returns to enable it. 193 * If we were to enable it in initarm then any authentication when 194 * returning would fail as it was called with pointer authentication 195 * disabled. 196 */ 197 bl ptrauth_start 198 199 bl mi_startup 200 201 /* We should not get here */ 202 brk 0 203 204 .align 3 205.Lvirtdone: 206 .quad virtdone 207.Lbss: 208 .quad __bss_start 209.Lend: 210 .quad __bss_end 211END(_start) 212 213#ifdef SMP 214/* 215 * void 216 * mpentry_psci(unsigned long) 217 * 218 * Called by a core when it is being brought online with psci. 219 * The data in x0 is passed straight to init_secondary. 220 */ 221ENTRY(mpentry_psci) 222 mov x26, xzr 223 b mpentry_common 224END(mpentry_psci) 225 226/* 227 * void 228 * mpentry_spintable(void) 229 * 230 * Called by a core when it is being brought online with a spin-table. 231 * Reads the new CPU ID and passes this to init_secondary. 232 */ 233ENTRY(mpentry_spintable) 234 ldr x26, =spintable_wait 235 b mpentry_common 236END(mpentry_spintable) 237 238/* Wait for the current CPU to be released */ 239LENTRY(spintable_wait) 240 /* Read the affinity bits from mpidr_el1 */ 241 mrs x1, mpidr_el1 242 ldr x2, =CPU_AFF_MASK 243 and x1, x1, x2 244 245 adrp x2, ap_cpuid 2461: 247 ldr x0, [x2, :lo12:ap_cpuid] 248 cmp x0, x1 249 b.ne 1b 250 251 str xzr, [x2, :lo12:ap_cpuid] 252 dsb sy 253 sev 254 255 ret 256LEND(mpentry_spintable) 257 258LENTRY(mpentry_common) 259 /* Disable interrupts */ 260 msr daifset, #DAIF_INTR 261 262 /* Enter the kernel exception level */ 263 bl enter_kernel_el 264 265 /* Set the context id */ 266 msr contextidr_el1, xzr 267 268 /* Load the kernel page table */ 269 adrp x24, pagetable_l0_ttbr1 270 add x24, x24, :lo12:pagetable_l0_ttbr1 271 /* Load the identity page table */ 272 adrp x27, pagetable_l0_ttbr0_bootstrap 273 add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 274 275 /* Enable the mmu */ 276 bl start_mmu 277 278 /* Load the new ttbr0 pagetable */ 279 adrp x27, pagetable_l0_ttbr0 280 add x27, x27, :lo12:pagetable_l0_ttbr0 281 282 /* Jump to the virtual address space */ 283 ldr x15, =mp_virtdone 284 br x15 285 286mp_virtdone: 287 BTI_J 288 289 /* 290 * Allow this CPU to wait until the kernel is ready for it, 291 * e.g. with spin-table but each CPU uses the same release address 292 */ 293 cbz x26, 1f 294 blr x26 2951: 296 297 /* Start using the AP boot stack */ 298 adrp x4, bootstack 299 ldr x4, [x4, :lo12:bootstack] 300 mov sp, x4 301 302#if defined(PERTHREAD_SSP) 303 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 304 adrp x15, boot_canary 305 add x15, x15, :lo12:boot_canary 306 msr sp_el0, x15 307#endif 308 309 /* Load the kernel ttbr0 pagetable */ 310 msr ttbr0_el1, x27 311 isb 312 313 /* Invalidate the TLB */ 314 tlbi vmalle1 315 dsb sy 316 isb 317 318 /* 319 * Initialize the per-CPU pointer before calling into C code, for the 320 * benefit of kernel sanitizers. 321 */ 322 adrp x18, bootpcpu 323 ldr x18, [x18, :lo12:bootpcpu] 324 msr tpidr_el1, x18 325 326 b init_secondary 327LEND(mpentry_common) 328 329ENTRY(mp_cpu_spinloop) 3300: 331 wfe 332 ldr x0, mp_cpu_spin_table_release_addr 333 cbz x0, 0b 334 blr x0 335 .globl mp_cpu_spin_table_release_addr 336mp_cpu_spin_table_release_addr: 337 .quad 0 338 .globl mp_cpu_spinloop_end 339mp_cpu_spinloop_end: 340END(mp_cpu_spinloop) 341#endif 342 343/* 344 * Enter the exception level the kernel will use: 345 * 346 * - If in EL1 continue in EL1 347 * - If the CPU supports FEAT_VHE then set HCR_E2H and HCR_TGE and continue 348 * in EL2 349 * - Configure EL2 to support running the kernel at EL1 and exit to that 350 */ 351LENTRY(enter_kernel_el) 352 mrs x23, CurrentEL 353 and x23, x23, #(CURRENTEL_EL_MASK) 354 cmp x23, #(CURRENTEL_EL_EL2) 355 b.eq 1f 356 357 /* 358 * Ensure there are no memory operations here. If the boot loader 359 * enters the kernel in big-endian mode then loading sctlr will 360 * be incorrect. As instructions are the same in both endians it is 361 * safe to use mov instructions. 362 */ 363 mov_q x2, SCTLR_MMU_OFF 364 msr sctlr_el1, x2 365 /* 366 * SCTLR_EOS is set to make eret a context synchronizing event. We 367 * need an isb here to ensure it's observed by later instructions, 368 * but don't need it in the eret below. 369 */ 370 isb 371 372 /* 373 * Ensure SPSR_EL1 and pstate are in sync. The only way to set the 374 * latter is to set the former and return from an exception with eret. 375 */ 376 mov x2, #(PSR_DAIF | PSR_M_EL1h) 377 msr spsr_el1, x2 378 msr elr_el1, lr 379 eret 380 3811: 382 dsb sy 383 /* 384 * Set just the reserved bits in sctlr_el2. This will disable the 385 * MMU which may have broken the kernel if we enter the kernel in 386 * EL2, e.g. when using VHE. 387 * 388 * As with sctlr_el1 above use mov instructions to ensure there are 389 * no memory operations. 390 */ 391 mov_q x2, (SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) 392 msr sctlr_el2, x2 393 isb 394 395 /* 396 * The hardware is now in little-endian mode so memory operations 397 * are safe. 398 */ 399 400 /* Configure the Hypervisor */ 401 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 402 msr hcr_el2, x2 403 404 /* Stash value of HCR_EL2 for later */ 405 isb 406 mrs x4, hcr_el2 407 408 /* Load the Virtualization Process ID Register */ 409 mrs x2, midr_el1 410 msr vpidr_el2, x2 411 412 /* Load the Virtualization Multiprocess ID Register */ 413 mrs x2, mpidr_el1 414 msr vmpidr_el2, x2 415 416 /* Set the initial sctlr_el1 */ 417 ldr x2, =SCTLR_MMU_OFF 418 msr sctlr_el1, x2 419 420 /* Check for VHE */ 421 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, VH, IMPL, .Lno_vhe) 422 423 /* 424 * The kernel will be running in EL2, route exceptions here rather 425 * than EL1. 426 */ 427 orr x4, x4, #HCR_E2H 428 orr x4, x4, #HCR_TGE 429 msr hcr_el2, x4 430 isb 431 432 msr SCTLR_EL12_REG, x2 433 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ 434 ldr x3, =(CNTHCTL_E2H_EL1PCTEN_NOTRAP | CNTHCTL_E2H_EL1PTEN_NOTRAP) 435 ldr x5, =(PSR_DAIF | PSR_M_EL2h) 436 b .Ldone_vhe 437 438.Lno_vhe: 439 /* Hypervisor trap functions */ 440 adrp x2, hyp_stub_vectors 441 add x2, x2, :lo12:hyp_stub_vectors 442 msr vbar_el2, x2 443 444 ldr x2, =(CPTR_RES1) 445 ldr x3, =(CNTHCTL_EL1PCTEN_NOTRAP | CNTHCTL_EL1PCEN_NOTRAP) 446 ldr x5, =(PSR_DAIF | PSR_M_EL1h) 447 448 /* Enable SPE at EL1 via Monitor Debug Configuration Register */ 449 mov x6, MDCR_EL2_E2PB_EL1_0_NO_TRAP 450 msr mdcr_el2, x6 451 452.Ldone_vhe: 453 454 msr cptr_el2, x2 455 /* Enable access to the physical timers at EL1 */ 456 msr cnthctl_el2, x3 457 /* Set the return PSTATE */ 458 msr spsr_el2, x5 459 460 /* 461 * Configure the Extended Hypervisor register. This is only valid if 462 * FEAT_HCX is enabled. 463 */ 464 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, HCX, IMPL, 2f) 465 /* Extended Hypervisor Configuration */ 466 msr HCRX_EL2_REG, xzr 467 isb 4682: 469 470 /* Don't trap to EL2 for CP15 traps */ 471 msr hstr_el2, xzr 472 473 /* Set the counter offset to a known value */ 474 msr cntvoff_el2, xzr 475 476 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 477 msr vttbr_el2, xzr 478 479 /* Check the CPU supports GIC, and configure the CPU interface */ 480 CHECK_CPU_FEAT(x2, ID_AA64PFR0, GIC, CPUIF_EN, 3f) 481 482 mrs x2, icc_sre_el2 483 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 484 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 485 msr icc_sre_el2, x2 4863: 487 488 /* Set the address to return to our return address */ 489 msr elr_el2, x30 490 isb 491 492 eret 493LEND(enter_kernel_el) 494 495/* Turn off the MMU. Install ttbr0 from the bootstrap page table, and go there. 496 * Does not return. 497 * - x0 - target address to jump to after stopping the MMU. 498 * - x1 - kernel load address 499 */ 500ENTRY(stop_mmu) 501 mov x16, x0 /* Save target. */ 502 ldr x2, =(1f - KERNBASE) 503 add x17, x1, x2 504 ldr x3, =(pagetable_l0_ttbr0_bootstrap - KERNBASE) 505 add x1, x1, x3 506 msr ttbr0_el1, x1 507 isb 508 br x17 5091: 510 BTI_J 511 mrs x0, sctlr_el1 512 bic x0, x0, SCTLR_M 513 bic x0, x0, SCTLR_C 514 msr sctlr_el1, x0 515 isb 516 br x16 517END(stop_mmu) 518/* 519 * Get the physical address the kernel was loaded at. 520 */ 521LENTRY(get_load_phys_addr) 522 /* Load the offset of get_load_phys_addr from KERNBASE */ 523 ldr x28, =(get_load_phys_addr - KERNBASE) 524 /* Load the physical address of get_load_phys_addr */ 525 adr x29, get_load_phys_addr 526 /* Find the physical address of KERNBASE, i.e. our load address */ 527 sub x28, x29, x28 528 ret 529LEND(get_load_phys_addr) 530 531/* 532 * This builds the page tables containing the identity map, and the kernel 533 * virtual map. 534 * 535 * It relys on: 536 * We were loaded to an address that is on a 2MiB boundary 537 * All the memory must not cross a 1GiB boundaty 538 * x28 contains the physical address we were loaded from 539 * 540 * There are 7 or 8 pages before that address for the page tables 541 * The pages used are: 542 * - The Kernel L3 tables (only for 16k kernel) 543 * - The Kernel L2 table 544 * - The Kernel L1 table 545 * - The Kernel L0 table (TTBR1) 546 * - The identity (PA = VA) L2 table 547 * - The identity (PA = VA) L1 table 548 * - The identity (PA = VA) L0 table (Early TTBR0) 549 * - The Kernel empty L0 table (Late TTBR0) 550 */ 551LENTRY(create_pagetables) 552 /* Save the Link register */ 553 mov x5, x30 554 555 /* Clean the page table */ 556 adrp x6, pagetable 557 add x6, x6, :lo12:pagetable 558 adrp x27, pagetable_end 559 add x27, x27, :lo12:pagetable_end 5601: 561 stp xzr, xzr, [x6], #16 562 stp xzr, xzr, [x6], #16 563 stp xzr, xzr, [x6], #16 564 stp xzr, xzr, [x6], #16 565 cmp x6, x27 566 b.lo 1b 567 568#ifdef __ARM_FEATURE_BTI_DEFAULT 569 /* 570 * Check if the CPU supports BTI 571 */ 572 mrs x6, id_aa64pfr1_el1 /* Read the ID register */ 573 and x6, x6, ID_AA64PFR1_BT_MASK /* Mask the field we need */ 574 cmp x6, xzr /* Check it's non-zero */ 575 cset x6, ne /* x6 is set if non-zero */ 576 lsl x21, x6, ATTR_S1_GP_SHIFT /* Shift to the correct bit */ 577#endif 578 579 /* 580 * Find the shareability attribute we should use. If FEAT_LPA2 is 581 * enabled then the shareability field is moved from the page table 582 * to tcr_el1 and the bits in the page table are reused by the 583 * address field. 584 */ 585#if PAGE_SIZE == PAGE_SIZE_4K 586#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK 587#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 588#elif PAGE_SIZE == PAGE_SIZE_16K 589#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK 590#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 591#else 592#error Unsupported page size 593#endif 594 mrs x6, id_aa64mmfr0_el1 595 mov x7, LPA2_VAL 596 and x6, x6, LPA2_MASK 597 cmp x6, x7 598 ldr x22, =(ATTR_SH(ATTR_SH_IS)) 599 csel x22, xzr, x22, eq 600#undef LPA2_MASK 601#undef LPA2_VAL 602 603 /* 604 * Build the TTBR1 maps. 605 */ 606 607 /* Find the size of the kernel */ 608 mov x6, #(KERNBASE) 609 610#if defined(LINUX_BOOT_ABI) 611 /* X19 is used as 'map FDT data' flag */ 612 mov x19, xzr 613 614 /* No modules or FDT pointer ? */ 615 cbz x0, booti_no_fdt 616 617 /* 618 * Test if x0 points to modules descriptor(virtual address) or 619 * to FDT (physical address) 620 */ 621 cmp x0, x6 /* x6 is #(KERNBASE) */ 622 b.lo booti_fdt 623#endif 624 625 /* Booted with modules pointer */ 626 /* Find modulep - begin */ 627 sub x8, x0, x6 628 /* 629 * Add space for the module data. When PAGE_SIZE is 4k this will 630 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 631 * larger it will be at least as large as we use smaller level 3 632 * pages. 633 */ 634 ldr x7, =((6 * 1024 * 1024) - 1) 635 add x8, x8, x7 636 b common 637 638#if defined(LINUX_BOOT_ABI) 639booti_fdt: 640 /* Booted by U-Boot booti with FDT data */ 641 /* Set 'map FDT data' flag */ 642 mov x19, #1 643 644booti_no_fdt: 645 /* Booted by U-Boot booti without FTD data */ 646 /* Find the end - begin */ 647 ldr x7, .Lend 648 sub x8, x7, x6 649 650 /* 651 * Add one 2MiB page for copy of FDT data (maximum FDT size), 652 * one for metadata and round up 653 */ 654 ldr x7, =(3 * L2_SIZE - 1) 655 add x8, x8, x7 656#endif 657 658common: 659#if PAGE_SIZE != PAGE_SIZE_4K 660 /* 661 * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned 662 * address, enabling the creation of L3C pages. However, when the page 663 * size is larger than 4k, L2 blocks are too large to map the kernel 664 * with 2M alignment. 665 */ 666#define PTE_SHIFT L3_SHIFT 667#define LL_PAGE_TABLE pagetable_l3_ttbr1 668#define BUILD_PTE_FUNC build_l3_page_pagetable 669#else 670#define PTE_SHIFT L2_SHIFT 671#define LL_PAGE_TABLE pagetable_l2_ttbr1 672#define BUILD_PTE_FUNC build_l2_block_pagetable 673#endif 674 675 /* Get the number of blocks/pages to allocate, rounded down */ 676 lsr x14, x8, #(PTE_SHIFT) 677 678 ldr x26, =etext 679#if PAGE_SIZE != PAGE_SIZE_4K 680 ldr x8, =((1 << PTE_SHIFT) - 1) 681 add x26, x26, x8 682#endif 683 mov x8, #(KERNBASE) 684 sub x25, x26, x8 685 lsr x25, x25, #(PTE_SHIFT) 686 687#if PAGE_SIZE == PAGE_SIZE_4K 688 /* Calculate the number of executable level 3 pages to create */ 689 lsr x26, x26, #(L3_SHIFT) 690 bfc x26, #(Ln_ENTRIES_SHIFT), #(64 - Ln_ENTRIES_SHIFT) 691 692 /* Build the L3 table holding the end of the exectuable code */ 693 lsl x15, x25, #(PTE_SHIFT) 694 adrp x6, pagetable_l3_ttbr1 695 add x6, x6, :lo12:pagetable_l3_ttbr1 696 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 697 ATTR_S1_AP(ATTR_S1_AP_RO)) 698 ldr x8, =(KERNBASE) 699 add x8, x8, x15 700 add x9, x28, x15 701 mov x10, x26 702 bl build_l3_page_pagetable 703 704 /* Build the remaining level 3 pages */ 705 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 706 lsl x27, x26, #(L3_SHIFT) 707 add x8, x8, x27 708 add x9, x28, x15 709 add x9, x9, x27 710 ldr x10, =(Ln_ENTRIES) 711 sub x10, x10, x26 712 bl build_l3_page_pagetable 713 714 /* Link the l2 -> l3 table */ 715 mov x9, x6 716 adrp x6, pagetable_l2_ttbr1 717 add x6, x6, :lo12:pagetable_l2_ttbr1 718 bl link_l2_pagetable 719#endif 720 721 /* Create the kernel space PTE table */ 722 adrp x6, LL_PAGE_TABLE 723 add x6, x6, :lo12:LL_PAGE_TABLE 724 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 725 ATTR_S1_AP(ATTR_S1_AP_RO)) 726 mov x8, #(KERNBASE) 727 mov x9, x28 728 mov x10, x25 729 bl BUILD_PTE_FUNC 730 731#if PAGE_SIZE == PAGE_SIZE_4K 732 /* Skip memory mapped through the L2 table */ 733 add x25, x25, #1 734#endif 735 736 /* Create the kernel space XN PTE table */ 737 lsl x10, x25, #(PTE_SHIFT) 738 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 739 ldr x8, =(KERNBASE) 740 add x8, x8, x10 741 add x9, x28, x10 742 sub x10, x14, x25 743 bl BUILD_PTE_FUNC 744 745#undef PTE_SHIFT 746#undef LL_PAGE_TABLE 747#undef BUILD_PTE_FUNC 748 749#if PAGE_SIZE != PAGE_SIZE_4K 750 /* Link the l2 -> l3 table */ 751 mov x9, x6 752 adrp x6, pagetable_l2_ttbr1 753 add x6, x6, :lo12:pagetable_l2_ttbr1 754 bl link_l2_pagetable 755#endif 756 757 /* Link the l1 -> l2 table */ 758 mov x9, x6 759 adrp x6, pagetable_l1_ttbr1 760 add x6, x6, :lo12:pagetable_l1_ttbr1 761 bl link_l1_pagetable 762 763 /* Link the l0 -> l1 table */ 764 mov x9, x6 765 adrp x6, pagetable_l0_ttbr1 766 add x6, x6, :lo12:pagetable_l0_ttbr1 767 mov x10, #1 768 bl link_l0_pagetable 769 770 /* Save the TTBR1 table physical address */ 771 mov x24, x6 772 773 /* 774 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 775 * They are only needed early on, so the VA = PA map is uncached. 776 */ 777 778 adrp x6, pagetable_l2_ttbr0_bootstrap 779 add x6, x6, :lo12:pagetable_l2_ttbr0_bootstrap 780 781 /* Create the VA = PA map */ 782 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 783 adrp x16, _start 784 and x16, x16, #(~L2_OFFSET) 785 mov x9, x16 /* PA start */ 786 mov x8, x16 /* VA start (== PA start) */ 787 mov x10, #1 788 bl build_l2_block_pagetable 789 790#if defined(SOCDEV_PA) 791 /* Create a table for the UART */ 792 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 793 ldr x9, =(L2_SIZE) 794 add x16, x16, x9 /* VA start */ 795 mov x8, x16 796 797 /* Store the socdev virtual address */ 798 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 799 adrp x9, socdev_va 800 str x17, [x9, :lo12:socdev_va] 801 802 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 803 mov x10, #1 804 bl build_l2_block_pagetable 805#endif 806 807#if defined(LINUX_BOOT_ABI) 808 /* Map FDT data ? */ 809 cbz x19, 1f 810 811 /* Create the mapping for FDT data (2 MiB max) */ 812 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 813 ldr x9, =(L2_SIZE) 814 add x16, x16, x9 /* VA start */ 815 mov x8, x16 816 mov x9, x0 /* PA start */ 817 /* Update the module pointer to point at the allocated memory */ 818 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 819 add x0, x0, x8 /* Add the aligned virtual address */ 820 821 mov x10, #1 822 bl build_l2_block_pagetable 823 8241: 825#endif 826 827 /* Link the l1 -> l2 table */ 828 mov x9, x6 829 adrp x6, pagetable_l1_ttbr0_bootstrap 830 add x6, x6, :lo12:pagetable_l1_ttbr0_bootstrap 831 bl link_l1_pagetable 832 833 /* Link the l0 -> l1 table */ 834 mov x9, x6 835 adrp x6, pagetable_l0_ttbr0_bootstrap 836 add x6, x6, :lo12:pagetable_l0_ttbr0_bootstrap 837 mov x10, #1 838 bl link_l0_pagetable 839 840 /* Save the TTBR0 table physical address */ 841 mov x27, x6 842 843 /* Restore the Link register */ 844 mov x30, x5 845 ret 846LEND(create_pagetables) 847 848/* 849 * Builds an L0 -> L1 table descriptor 850 * 851 * x6 = L0 table 852 * x8 = Virtual Address 853 * x9 = L1 PA (trashed) 854 * x10 = Entry count (trashed) 855 * x11, x12 and x13 are trashed 856 */ 857LENTRY(link_l0_pagetable) 858 /* 859 * Link an L0 -> L1 table entry. 860 */ 861 /* Find the table index */ 862 lsr x11, x8, #L0_SHIFT 863 and x11, x11, #L0_ADDR_MASK 864 865 /* Build the L0 block entry */ 866 mov x12, #L0_TABLE 867 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 868 869 /* Only use the output address bits */ 870 lsr x9, x9, #PAGE_SHIFT 8711: orr x13, x12, x9, lsl #PAGE_SHIFT 872 873 /* Store the entry */ 874 str x13, [x6, x11, lsl #3] 875 876 sub x10, x10, #1 877 add x11, x11, #1 878 add x9, x9, #1 879 cbnz x10, 1b 880 881 ret 882LEND(link_l0_pagetable) 883 884/* 885 * Builds an L1 -> L2 table descriptor 886 * 887 * x6 = L1 table 888 * x8 = Virtual Address 889 * x9 = L2 PA (trashed) 890 * x11, x12 and x13 are trashed 891 */ 892LENTRY(link_l1_pagetable) 893 /* 894 * Link an L1 -> L2 table entry. 895 */ 896 /* Find the table index */ 897 lsr x11, x8, #L1_SHIFT 898 and x11, x11, #Ln_ADDR_MASK 899 900 /* Build the L1 block entry */ 901 mov x12, #L1_TABLE 902 903 /* Only use the output address bits */ 904 lsr x9, x9, #PAGE_SHIFT 905 orr x13, x12, x9, lsl #PAGE_SHIFT 906 907 /* Store the entry */ 908 str x13, [x6, x11, lsl #3] 909 910 ret 911LEND(link_l1_pagetable) 912 913/* 914 * Builds count 2 MiB page table entry 915 * x6 = L2 table 916 * x7 = Block attributes 917 * x8 = VA start 918 * x9 = PA start (trashed) 919 * x10 = Entry count (trashed) 920 * x11, x12 and x13 are trashed 921 */ 922LENTRY(build_l2_block_pagetable) 923 /* 924 * Build the L2 table entry. 925 */ 926 /* Find the table index */ 927 lsr x11, x8, #L2_SHIFT 928 and x11, x11, #Ln_ADDR_MASK 929 930 /* Build the L2 block entry */ 931 orr x12, x7, #L2_BLOCK 932 orr x12, x12, #(ATTR_AF) 933 orr x12, x12, #(ATTR_S1_UXN) 934#ifdef __ARM_FEATURE_BTI_DEFAULT 935 orr x12, x12, x21 936#endif 937 /* Set the shareability attribute */ 938 orr x12, x12, x22 939 940 /* Only use the output address bits */ 941 lsr x9, x9, #L2_SHIFT 942 943 /* Set the physical address for this virtual address */ 9441: orr x13, x12, x9, lsl #L2_SHIFT 945 946 /* Store the entry */ 947 str x13, [x6, x11, lsl #3] 948 949 sub x10, x10, #1 950 add x11, x11, #1 951 add x9, x9, #1 952 cbnz x10, 1b 953 954 ret 955LEND(build_l2_block_pagetable) 956 957/* 958 * Builds an L2 -> L3 table descriptor 959 * 960 * x6 = L2 table 961 * x8 = Virtual Address 962 * x9 = L3 PA (trashed) 963 * x11, x12 and x13 are trashed 964 */ 965LENTRY(link_l2_pagetable) 966 /* 967 * Link an L2 -> L3 table entry. 968 */ 969 /* Find the table index */ 970 lsr x11, x8, #L2_SHIFT 971 and x11, x11, #Ln_ADDR_MASK 972 973 /* Build the L1 block entry */ 974 mov x12, #L2_TABLE 975 976 /* Only use the output address bits */ 977 lsr x9, x9, #PAGE_SHIFT 978 orr x13, x12, x9, lsl #PAGE_SHIFT 979 980 /* Store the entry */ 981 str x13, [x6, x11, lsl #3] 982 983 ret 984LEND(link_l2_pagetable) 985 986/* 987 * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create 988 * large page (L3C) mappings when the current VA and remaining count allow 989 * it. 990 * x6 = L3 table 991 * x7 = Block attributes 992 * x8 = VA start 993 * x9 = PA start (trashed) 994 * x10 = Entry count (trashed) 995 * x11, x12 and x13 are trashed 996 * 997 * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. 998 */ 999LENTRY(build_l3_page_pagetable) 1000 cbz x10, 2f 1001 /* 1002 * Build the L3 table entry. 1003 */ 1004 /* Find the table index */ 1005 lsr x11, x8, #L3_SHIFT 1006 and x11, x11, #Ln_ADDR_MASK 1007 1008 /* Build the L3 page entry */ 1009 orr x12, x7, #L3_PAGE 1010 orr x12, x12, #(ATTR_AF) 1011 orr x12, x12, #(ATTR_S1_UXN) 1012#ifdef __ARM_FEATURE_BTI_DEFAULT 1013 orr x12, x12, x21 1014#endif 1015 /* Set the shareability attribute */ 1016 orr x12, x12, x22 1017 1018 /* Only use the output address bits */ 1019 lsr x9, x9, #L3_SHIFT 1020 1021 /* Check if an ATTR_CONTIGUOUS mapping is possible */ 10221: tst x11, #(L3C_ENTRIES - 1) 1023 b.ne 2f 1024 cmp x10, #L3C_ENTRIES 1025 b.lo 3f 1026 orr x12, x12, #(ATTR_CONTIGUOUS) 1027 b 2f 10283: and x12, x12, #(~ATTR_CONTIGUOUS) 1029 1030 /* Set the physical address for this virtual address */ 10312: orr x13, x12, x9, lsl #L3_SHIFT 1032 1033 /* Store the entry */ 1034 str x13, [x6, x11, lsl #3] 1035 1036 sub x10, x10, #1 1037 add x11, x11, #1 1038 add x9, x9, #1 1039 cbnz x10, 1b 10402: 1041 1042 ret 1043LEND(build_l3_page_pagetable) 1044 1045LENTRY(start_mmu) 1046 dsb sy 1047 1048 /* Load the exception vectors */ 1049 ldr x2, =exception_vectors 1050 msr vbar_el1, x2 1051 1052 /* Load ttbr0 and ttbr1 */ 1053 msr ttbr0_el1, x27 1054 msr ttbr1_el1, x24 1055 isb 1056 1057 /* Clear the Monitor Debug System control register */ 1058 msr mdscr_el1, xzr 1059 1060 /* Invalidate the TLB */ 1061 tlbi vmalle1is 1062 dsb ish 1063 isb 1064 1065 ldr x2, mair 1066 msr mair_el1, x2 1067 1068 /* 1069 * Setup TCR according to the PARange and ASIDBits fields 1070 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 1071 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 1072 * to 1 only if the ASIDBits field equals 0b0010. 1073 */ 1074 ldr x2, tcr 1075 1076 /* If x22 contains a non-zero value then LPA2 is not implemented */ 1077 cbnz x22, .Lno_lpa2 1078 ldr x3, =(TCR_DS) 1079 orr x2, x2, x3 1080.Lno_lpa2: 1081 1082 mrs x3, id_aa64mmfr0_el1 1083 1084 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 1085 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 1086 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 1087 1088 /* Check if the HW supports 16 bit ASIDS */ 1089 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 1090 /* If so x3 == 1, else x3 == 0 */ 1091 cset x3, eq 1092 /* Set TCR.AS with x3 */ 1093 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 1094 1095 /* 1096 * Check if the HW supports access flag updates, and set 1097 * TCR_EL1.HA accordingly. The TCR_EL1.HD flag to enable 1098 * HW management of dirty state is set in C code as it may 1099 * need to be disabled because of CPU errata. 1100 */ 1101 CHECK_CPU_FEAT(x3, ID_AA64MMFR1, HAFDBS, AF, 1f) 1102 orr x2, x2, #(TCR_HA) 11031: 1104 1105 msr tcr_el1, x2 1106 1107 /* 1108 * Setup SCTLR. 1109 */ 1110 ldr x1, =SCTLR_MMU_ON 1111 msr sctlr_el1, x1 1112 isb 1113 1114 ret 1115 1116 .align 3 1117mair: 1118 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 1119 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 1120 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 1121 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 1122 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 1123tcr: 1124#if PAGE_SIZE == PAGE_SIZE_4K 1125#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 1126#elif PAGE_SIZE == PAGE_SIZE_16K 1127#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 1128#else 1129#error Unsupported page size 1130#endif 1131 1132 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 1133 TCR_SH1_IS | TCR_ORGN1_WBWA | TCR_IRGN1_WBWA | \ 1134 TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) 1135LEND(start_mmu) 1136 1137ENTRY(switch_stack) 1138 mov sp, x0 1139 mov x16, x1 1140 br x16 1141END(switch_stack) 1142 1143ENTRY(abort) 1144 b abort 1145END(abort) 1146 1147.bss 1148 .align PAGE_SHIFT 1149 .globl initstack_end 1150initstack: 1151 .space BOOT_STACK_SIZE 1152initstack_end: 1153 1154 .section .init_pagetable, "aw", %nobits 1155 .align PAGE_SHIFT 1156 /* 1157 * 6 initial tables (in the following order): 1158 * L2 for kernel (High addresses) 1159 * L1 for kernel 1160 * L0 for kernel 1161 * L1 bootstrap for user (Low addresses) 1162 * L0 bootstrap for user 1163 * L0 for user 1164 */ 1165 .globl pagetable_l0_ttbr1 1166 .globl pagetable_l0_ttbr0_bootstrap 1167pagetable: 1168pagetable_l3_ttbr1: 1169 .space (PAGE_SIZE * L3_PAGE_COUNT) 1170pagetable_l2_ttbr1: 1171 .space PAGE_SIZE 1172pagetable_l1_ttbr1: 1173 .space PAGE_SIZE 1174pagetable_l0_ttbr1: 1175 .space PAGE_SIZE 1176pagetable_l2_ttbr0_bootstrap: 1177 .space PAGE_SIZE 1178pagetable_l1_ttbr0_bootstrap: 1179 .space PAGE_SIZE 1180pagetable_l0_ttbr0_bootstrap: 1181 .space PAGE_SIZE 1182pagetable_l0_ttbr0: 1183 .space PAGE_SIZE 1184pagetable_end: 1185 1186el2_pagetable: 1187 .space PAGE_SIZE 1188 1189 .section .rodata, "a", %progbits 1190 .globl aarch32_sigcode 1191 .align 2 1192aarch32_sigcode: 1193 .word 0xe1a0000d // mov r0, sp 1194 .word 0xe2800040 // add r0, r0, #SIGF_UC 1195 .word 0xe59f700c // ldr r7, [pc, #12] 1196 .word 0xef000000 // swi #0 1197 .word 0xe59f7008 // ldr r7, [pc, #8] 1198 .word 0xef000000 // swi #0 1199 .word 0xeafffffa // b . - 16 1200 .word SYS_sigreturn 1201 .word SYS_exit 1202 .align 3 1203 .size aarch32_sigcode, . - aarch32_sigcode 1204aarch32_esigcode: 1205 .data 1206 .global sz_aarch32_sigcode 1207sz_aarch32_sigcode: 1208 .quad aarch32_esigcode - aarch32_sigcode 1209 1210GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 1211