1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/elf_common.h> 30#include <sys/syscall.h> 31#include <machine/asm.h> 32#include <machine/armreg.h> 33#include <machine/cpu.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42#if PAGE_SIZE == PAGE_SIZE_16K 43/* 44 * The number of level 3 tables to create. 32 will allow for 1G of address 45 * space, the same as a single level 2 page with 4k pages. 46 */ 47#define L3_PAGE_COUNT 32 48#elif PAGE_SIZE == PAGE_SIZE_4K 49/* 50 * Space for a level 3 table holding the end of the executable memory and 51 * the start of the non-executable data. 52 */ 53#define L3_PAGE_COUNT 1 54#endif 55 56/* 57 * The size of our bootstrap stack. 58 */ 59#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 60 61 .globl kernbase 62 .set kernbase, KERNBASE 63 64/* 65 * We assume: 66 * MMU on with an identity map, or off 67 * D-Cache: off 68 * I-Cache: on or off 69 * We are loaded at a 2MiB aligned address 70 */ 71 72ENTRY(_start) 73 /* Enter the kernel exception level */ 74 bl enter_kernel_el 75 76 /* Set the context id */ 77 msr contextidr_el1, xzr 78 79 /* Get the virt -> phys offset */ 80 bl get_load_phys_addr 81 82 /* 83 * At this point: 84 * x28 = Our physical load address 85 */ 86 87 /* Create the page tables */ 88 bl create_pagetables 89 90 /* 91 * At this point: 92 * x27 = TTBR0 table 93 * x24 = TTBR1 table 94 * x22 = PTE shareability attributes 95 * x21 = BTI guarded page attribute if supported 96 */ 97 98 /* Enable the mmu */ 99 bl start_mmu 100 101 /* Load the new ttbr0 pagetable */ 102 adrp x27, pagetable_l0_ttbr0 103 add x27, x27, :lo12:pagetable_l0_ttbr0 104 105 /* Jump to the virtual address space */ 106 ldr x15, .Lvirtdone 107 br x15 108 109virtdone: 110 BTI_J 111 112 /* Set up the stack */ 113 adrp x25, initstack_end 114 add x25, x25, :lo12:initstack_end 115 sub sp, x25, #PCB_SIZE 116 117 /* Zero the BSS */ 118 ldr x15, .Lbss 119 ldr x14, .Lend 1201: 121 stp xzr, xzr, [x15], #16 122 cmp x15, x14 123 b.lo 1b 124 125#if defined(PERTHREAD_SSP) 126 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 127 adrp x15, boot_canary 128 add x15, x15, :lo12:boot_canary 129 msr sp_el0, x15 130#endif 131 132 /* Backup the module pointer */ 133 mov x1, x0 134 135 sub sp, sp, #BOOTPARAMS_SIZE 136 mov x0, sp 137 138 str x1, [x0, #BP_MODULEP] 139 adrp x25, initstack 140 add x25, x25, :lo12:initstack 141 str x25, [x0, #BP_KERN_STACK] 142 str x27, [x0, #BP_KERN_TTBR0] 143 str x23, [x0, #BP_BOOT_EL] 144 145 /* Set these before they are used in kasan_init_early */ 146 adrp x1, pmap_sh_attr 147 str x22, [x1, :lo12:pmap_sh_attr] 148#ifdef __ARM_FEATURE_BTI_DEFAULT 149 adrp x1, pmap_gp_attr 150 str x21, [x1, :lo12:pmap_gp_attr] 151#endif 152 153#ifdef KASAN 154 /* Save bootparams */ 155 mov x19, x0 156 157 /* Bootstrap an early shadow map for the boot stack. */ 158 ldr x0, [x0, #BP_KERN_STACK] 159 ldr x1, =BOOT_STACK_SIZE 160 bl kasan_init_early 161 162 /* Restore bootparams */ 163 mov x0, x19 164#endif 165 166 /* trace back starts here */ 167 mov fp, #0 168 /* Branch to C code */ 169 bl initarm 170 /* We are done with the boot params */ 171 add sp, sp, #BOOTPARAMS_SIZE 172 173 /* 174 * Enable pointer authentication in the kernel. We set the keys for 175 * thread0 in initarm so have to wait until it returns to enable it. 176 * If we were to enable it in initarm then any authentication when 177 * returning would fail as it was called with pointer authentication 178 * disabled. 179 */ 180 bl ptrauth_start 181 182 bl mi_startup 183 184 /* We should not get here */ 185 brk 0 186 187 .align 3 188.Lvirtdone: 189 .quad virtdone 190.Lbss: 191 .quad __bss_start 192.Lend: 193 .quad __bss_end 194END(_start) 195 196#ifdef SMP 197/* 198 * void 199 * mpentry_psci(unsigned long) 200 * 201 * Called by a core when it is being brought online with psci. 202 * The data in x0 is passed straight to init_secondary. 203 */ 204ENTRY(mpentry_psci) 205 mov x26, xzr 206 b mpentry_common 207END(mpentry_psci) 208 209/* 210 * void 211 * mpentry_spintable(void) 212 * 213 * Called by a core when it is being brought online with a spin-table. 214 * Reads the new CPU ID and passes this to init_secondary. 215 */ 216ENTRY(mpentry_spintable) 217 ldr x26, =spintable_wait 218 b mpentry_common 219END(mpentry_spintable) 220 221/* Wait for the current CPU to be released */ 222LENTRY(spintable_wait) 223 /* Read the affinity bits from mpidr_el1 */ 224 mrs x1, mpidr_el1 225 ldr x2, =CPU_AFF_MASK 226 and x1, x1, x2 227 228 adrp x2, ap_cpuid 2291: 230 ldr x0, [x2, :lo12:ap_cpuid] 231 cmp x0, x1 232 b.ne 1b 233 234 str xzr, [x2, :lo12:ap_cpuid] 235 dsb sy 236 sev 237 238 ret 239LEND(mpentry_spintable) 240 241LENTRY(mpentry_common) 242 /* Disable interrupts */ 243 msr daifset, #DAIF_INTR 244 245 /* Enter the kernel exception level */ 246 bl enter_kernel_el 247 248 /* Set the context id */ 249 msr contextidr_el1, xzr 250 251 /* Load the kernel page table */ 252 adrp x24, pagetable_l0_ttbr1 253 add x24, x24, :lo12:pagetable_l0_ttbr1 254 /* Load the identity page table */ 255 adrp x27, pagetable_l0_ttbr0_bootstrap 256 add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 257 258 /* Enable the mmu */ 259 bl start_mmu 260 261 /* Load the new ttbr0 pagetable */ 262 adrp x27, pagetable_l0_ttbr0 263 add x27, x27, :lo12:pagetable_l0_ttbr0 264 265 /* Jump to the virtual address space */ 266 ldr x15, =mp_virtdone 267 br x15 268 269mp_virtdone: 270 BTI_J 271 272 /* 273 * Allow this CPU to wait until the kernel is ready for it, 274 * e.g. with spin-table but each CPU uses the same release address 275 */ 276 cbz x26, 1f 277 blr x26 2781: 279 280 /* Start using the AP boot stack */ 281 adrp x4, bootstack 282 ldr x4, [x4, :lo12:bootstack] 283 mov sp, x4 284 285#if defined(PERTHREAD_SSP) 286 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 287 adrp x15, boot_canary 288 add x15, x15, :lo12:boot_canary 289 msr sp_el0, x15 290#endif 291 292 /* Load the kernel ttbr0 pagetable */ 293 msr ttbr0_el1, x27 294 isb 295 296 /* Invalidate the TLB */ 297 tlbi vmalle1 298 dsb sy 299 isb 300 301 /* 302 * Initialize the per-CPU pointer before calling into C code, for the 303 * benefit of kernel sanitizers. 304 */ 305 adrp x18, bootpcpu 306 ldr x18, [x18, :lo12:bootpcpu] 307 msr tpidr_el1, x18 308 309 b init_secondary 310LEND(mpentry_common) 311#endif 312 313/* 314 * Enter the exception level the kernel will use: 315 * 316 * - If in EL1 continue in EL1 317 * - If the CPU supports FEAT_VHE then set HCR_E2H and HCR_TGE and continue 318 * in EL2 319 * - Configure EL2 to support running the kernel at EL1 and exit to that 320 */ 321LENTRY(enter_kernel_el) 322#define INIT_SCTLR_EL1 (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_EIS | \ 323 SCTLR_TSCXT | SCTLR_EOS) 324 mrs x23, CurrentEL 325 and x23, x23, #(CURRENTEL_EL_MASK) 326 cmp x23, #(CURRENTEL_EL_EL2) 327 b.eq 1f 328 329 ldr x2, =INIT_SCTLR_EL1 330 msr sctlr_el1, x2 331 /* SCTLR_EOS is set so eret is a context synchronizing event so we 332 * need an isb here to ensure it's observed by later instructions, 333 * but don't need it in the eret below. 334 */ 335 isb 336 337 /* Ensure SPSR_EL1 and pstate are in sync. The only wat to set the 338 * latter is to set the former and return from an exception with eret. 339 */ 340 mov x2, #(PSR_DAIF | PSR_M_EL1h) 341 msr spsr_el1, x2 342 msr elr_el1, lr 343 eret 344 3451: 346 dsb sy 347 /* 348 * Set just the reserved bits in sctlr_el2. This will disable the 349 * MMU which may have broken the kernel if we enter the kernel in 350 * EL2, e.g. when using VHE. 351 */ 352 ldr x2, =(SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) 353 msr sctlr_el2, x2 354 isb 355 356 /* Configure the Hypervisor */ 357 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 358 msr hcr_el2, x2 359 360 /* Stash value of HCR_EL2 for later */ 361 isb 362 mrs x4, hcr_el2 363 364 /* Load the Virtualization Process ID Register */ 365 mrs x2, midr_el1 366 msr vpidr_el2, x2 367 368 /* Load the Virtualization Multiprocess ID Register */ 369 mrs x2, mpidr_el1 370 msr vmpidr_el2, x2 371 372 /* Set the initial sctlr_el1 */ 373 ldr x2, =INIT_SCTLR_EL1 374 msr sctlr_el1, x2 375 376 /* Check for VHE */ 377 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, VH, IMPL, .Lno_vhe) 378 379 /* 380 * The kernel will be running in EL2, route exceptions here rather 381 * than EL1. 382 */ 383 orr x4, x4, #HCR_E2H 384 orr x4, x4, #HCR_TGE 385 msr hcr_el2, x4 386 isb 387 388 msr SCTLR_EL12_REG, x2 389 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ 390 ldr x3, =(CNTHCTL_E2H_EL1PCTEN | CNTHCTL_E2H_EL1PTEN) 391 ldr x5, =(PSR_DAIF | PSR_M_EL2h) 392 b .Ldone_vhe 393 394.Lno_vhe: 395 /* Hypervisor trap functions */ 396 adrp x2, hyp_stub_vectors 397 add x2, x2, :lo12:hyp_stub_vectors 398 msr vbar_el2, x2 399 400 ldr x2, =(CPTR_RES1) 401 ldr x3, =(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 402 ldr x5, =(PSR_DAIF | PSR_M_EL1h) 403 404.Ldone_vhe: 405 406 msr cptr_el2, x2 407 /* Enable access to the physical timers at EL1 */ 408 msr cnthctl_el2, x3 409 /* Set the return PSTATE */ 410 msr spsr_el2, x5 411 412 /* 413 * Configure the Extended Hypervisor register. This is only valid if 414 * FEAT_HCX is enabled. 415 */ 416 CHECK_CPU_FEAT(x2, ID_AA64MMFR1, HCX, IMPL, 2f) 417 /* Extended Hypervisor Configuration */ 418 msr HCRX_EL2_REG, xzr 419 isb 4202: 421 422 /* Don't trap to EL2 for CP15 traps */ 423 msr hstr_el2, xzr 424 425 /* Set the counter offset to a known value */ 426 msr cntvoff_el2, xzr 427 428 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 429 msr vttbr_el2, xzr 430 431 /* Check the CPU supports GIC, and configure the CPU interface */ 432 CHECK_CPU_FEAT(x2, ID_AA64PFR0, GIC, CPUIF_EN, 3f) 433 434 mrs x2, icc_sre_el2 435 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 436 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 437 msr icc_sre_el2, x2 4383: 439 440 /* Set the address to return to our return address */ 441 msr elr_el2, x30 442 isb 443 444 eret 445#undef INIT_SCTLR_EL1 446LEND(enter_kernel_el) 447 448/* 449 * Get the physical address the kernel was loaded at. 450 */ 451LENTRY(get_load_phys_addr) 452 /* Load the offset of get_load_phys_addr from KERNBASE */ 453 ldr x28, =(get_load_phys_addr - KERNBASE) 454 /* Load the physical address of get_load_phys_addr */ 455 adr x29, get_load_phys_addr 456 /* Find the physical address of KERNBASE, i.e. our load address */ 457 sub x28, x29, x28 458 ret 459LEND(get_load_phys_addr) 460 461/* 462 * This builds the page tables containing the identity map, and the kernel 463 * virtual map. 464 * 465 * It relys on: 466 * We were loaded to an address that is on a 2MiB boundary 467 * All the memory must not cross a 1GiB boundaty 468 * x28 contains the physical address we were loaded from 469 * 470 * There are 7 or 8 pages before that address for the page tables 471 * The pages used are: 472 * - The Kernel L3 tables (only for 16k kernel) 473 * - The Kernel L2 table 474 * - The Kernel L1 table 475 * - The Kernel L0 table (TTBR1) 476 * - The identity (PA = VA) L2 table 477 * - The identity (PA = VA) L1 table 478 * - The identity (PA = VA) L0 table (Early TTBR0) 479 * - The Kernel empty L0 table (Late TTBR0) 480 */ 481LENTRY(create_pagetables) 482 /* Save the Link register */ 483 mov x5, x30 484 485 /* Clean the page table */ 486 adrp x6, pagetable 487 add x6, x6, :lo12:pagetable 488 adrp x27, pagetable_end 489 add x27, x27, :lo12:pagetable_end 4901: 491 stp xzr, xzr, [x6], #16 492 stp xzr, xzr, [x6], #16 493 stp xzr, xzr, [x6], #16 494 stp xzr, xzr, [x6], #16 495 cmp x6, x27 496 b.lo 1b 497 498#ifdef __ARM_FEATURE_BTI_DEFAULT 499 /* 500 * Check if the CPU supports BTI 501 */ 502 mrs x6, id_aa64pfr1_el1 /* Read the ID register */ 503 and x6, x6, ID_AA64PFR1_BT_MASK /* Mask the field we need */ 504 cmp x6, xzr /* Check it's non-zero */ 505 cset x6, ne /* x6 is set if non-zero */ 506 lsl x21, x6, ATTR_S1_GP_SHIFT /* Shift to the correct bit */ 507#endif 508 509 /* 510 * Find the shareability attribute we should use. If FEAT_LPA2 is 511 * enabled then the shareability field is moved from the page table 512 * to tcr_el1 and the bits in the page table are reused by the 513 * address field. 514 */ 515#if PAGE_SIZE == PAGE_SIZE_4K 516#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK 517#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 518#elif PAGE_SIZE == PAGE_SIZE_16K 519#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK 520#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 521#else 522#error Unsupported page size 523#endif 524 mrs x6, id_aa64mmfr0_el1 525 mov x7, LPA2_VAL 526 and x6, x6, LPA2_MASK 527 cmp x6, x7 528 ldr x22, =(ATTR_SH(ATTR_SH_IS)) 529 csel x22, xzr, x22, eq 530#undef LPA2_MASK 531#undef LPA2_VAL 532 533 /* 534 * Build the TTBR1 maps. 535 */ 536 537 /* Find the size of the kernel */ 538 mov x6, #(KERNBASE) 539 540#if defined(LINUX_BOOT_ABI) 541 /* X19 is used as 'map FDT data' flag */ 542 mov x19, xzr 543 544 /* No modules or FDT pointer ? */ 545 cbz x0, booti_no_fdt 546 547 /* 548 * Test if x0 points to modules descriptor(virtual address) or 549 * to FDT (physical address) 550 */ 551 cmp x0, x6 /* x6 is #(KERNBASE) */ 552 b.lo booti_fdt 553#endif 554 555 /* Booted with modules pointer */ 556 /* Find modulep - begin */ 557 sub x8, x0, x6 558 /* 559 * Add space for the module data. When PAGE_SIZE is 4k this will 560 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 561 * larger it will be at least as large as we use smaller level 3 562 * pages. 563 */ 564 ldr x7, =((6 * 1024 * 1024) - 1) 565 add x8, x8, x7 566 b common 567 568#if defined(LINUX_BOOT_ABI) 569booti_fdt: 570 /* Booted by U-Boot booti with FDT data */ 571 /* Set 'map FDT data' flag */ 572 mov x19, #1 573 574booti_no_fdt: 575 /* Booted by U-Boot booti without FTD data */ 576 /* Find the end - begin */ 577 ldr x7, .Lend 578 sub x8, x7, x6 579 580 /* 581 * Add one 2MiB page for copy of FDT data (maximum FDT size), 582 * one for metadata and round up 583 */ 584 ldr x7, =(3 * L2_SIZE - 1) 585 add x8, x8, x7 586#endif 587 588common: 589#if PAGE_SIZE != PAGE_SIZE_4K 590 /* 591 * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned 592 * address, enabling the creation of L3C pages. However, when the page 593 * size is larger than 4k, L2 blocks are too large to map the kernel 594 * with 2M alignment. 595 */ 596#define PTE_SHIFT L3_SHIFT 597#define LL_PAGE_TABLE pagetable_l3_ttbr1 598#define BUILD_PTE_FUNC build_l3_page_pagetable 599#else 600#define PTE_SHIFT L2_SHIFT 601#define LL_PAGE_TABLE pagetable_l2_ttbr1 602#define BUILD_PTE_FUNC build_l2_block_pagetable 603#endif 604 605 /* Get the number of blocks/pages to allocate, rounded down */ 606 lsr x14, x8, #(PTE_SHIFT) 607 608 ldr x26, =etext 609#if PAGE_SIZE != PAGE_SIZE_4K 610 ldr x8, =((1 << PTE_SHIFT) - 1) 611 add x26, x26, x8 612#endif 613 mov x8, #(KERNBASE) 614 sub x25, x26, x8 615 lsr x25, x25, #(PTE_SHIFT) 616 617#if PAGE_SIZE == PAGE_SIZE_4K 618 /* Calculate the number of executable level 3 pages to create */ 619 lsr x26, x26, #(L3_SHIFT) 620 bfc x26, #(Ln_ENTRIES_SHIFT), #(64 - Ln_ENTRIES_SHIFT) 621 622 /* Build the L3 table holding the end of the exectuable code */ 623 lsl x15, x25, #(PTE_SHIFT) 624 adrp x6, pagetable_l3_ttbr1 625 add x6, x6, :lo12:pagetable_l3_ttbr1 626 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 627 ATTR_S1_AP(ATTR_S1_AP_RO)) 628 ldr x8, =(KERNBASE) 629 add x8, x8, x15 630 add x9, x28, x15 631 mov x10, x26 632 bl build_l3_page_pagetable 633 634 /* Build the remaining level 3 pages */ 635 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 636 lsl x27, x26, #(L3_SHIFT) 637 add x8, x8, x27 638 add x9, x28, x15 639 add x9, x9, x27 640 ldr x10, =(Ln_ENTRIES) 641 sub x10, x10, x26 642 bl build_l3_page_pagetable 643 644 /* Link the l2 -> l3 table */ 645 mov x9, x6 646 adrp x6, pagetable_l2_ttbr1 647 add x6, x6, :lo12:pagetable_l2_ttbr1 648 bl link_l2_pagetable 649#endif 650 651 /* Create the kernel space PTE table */ 652 adrp x6, LL_PAGE_TABLE 653 add x6, x6, :lo12:LL_PAGE_TABLE 654 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | \ 655 ATTR_S1_AP(ATTR_S1_AP_RO)) 656 mov x8, #(KERNBASE) 657 mov x9, x28 658 mov x10, x25 659 bl BUILD_PTE_FUNC 660 661#if PAGE_SIZE == PAGE_SIZE_4K 662 /* Skip memory mapped through the L2 table */ 663 add x25, x25, #1 664#endif 665 666 /* Create the kernel space XN PTE table */ 667 lsl x10, x25, #(PTE_SHIFT) 668 ldr x7, =(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | ATTR_S1_XN) 669 ldr x8, =(KERNBASE) 670 add x8, x8, x10 671 add x9, x28, x10 672 sub x10, x14, x25 673 bl BUILD_PTE_FUNC 674 675#undef PTE_SHIFT 676#undef LL_PAGE_TABLE 677#undef BUILD_PTE_FUNC 678 679#if PAGE_SIZE != PAGE_SIZE_4K 680 /* Link the l2 -> l3 table */ 681 mov x9, x6 682 adrp x6, pagetable_l2_ttbr1 683 add x6, x6, :lo12:pagetable_l2_ttbr1 684 bl link_l2_pagetable 685#endif 686 687 /* Link the l1 -> l2 table */ 688 mov x9, x6 689 adrp x6, pagetable_l1_ttbr1 690 add x6, x6, :lo12:pagetable_l1_ttbr1 691 bl link_l1_pagetable 692 693 /* Link the l0 -> l1 table */ 694 mov x9, x6 695 adrp x6, pagetable_l0_ttbr1 696 add x6, x6, :lo12:pagetable_l0_ttbr1 697 mov x10, #1 698 bl link_l0_pagetable 699 700 /* Save the TTBR1 table physical address */ 701 mov x24, x6 702 703 /* 704 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 705 * They are only needed early on, so the VA = PA map is uncached. 706 */ 707 708 adrp x6, pagetable_l2_ttbr0_bootstrap 709 add x6, x6, :lo12:pagetable_l2_ttbr0_bootstrap 710 711 /* Create the VA = PA map */ 712 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 713 adrp x16, _start 714 and x16, x16, #(~L2_OFFSET) 715 mov x9, x16 /* PA start */ 716 mov x8, x16 /* VA start (== PA start) */ 717 mov x10, #1 718 bl build_l2_block_pagetable 719 720#if defined(SOCDEV_PA) 721 /* Create a table for the UART */ 722 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 723 ldr x9, =(L2_SIZE) 724 add x16, x16, x9 /* VA start */ 725 mov x8, x16 726 727 /* Store the socdev virtual address */ 728 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 729 adrp x9, socdev_va 730 str x17, [x9, :lo12:socdev_va] 731 732 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 733 mov x10, #1 734 bl build_l2_block_pagetable 735#endif 736 737#if defined(LINUX_BOOT_ABI) 738 /* Map FDT data ? */ 739 cbz x19, 1f 740 741 /* Create the mapping for FDT data (2 MiB max) */ 742 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 743 ldr x9, =(L2_SIZE) 744 add x16, x16, x9 /* VA start */ 745 mov x8, x16 746 mov x9, x0 /* PA start */ 747 /* Update the module pointer to point at the allocated memory */ 748 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 749 add x0, x0, x8 /* Add the aligned virtual address */ 750 751 mov x10, #1 752 bl build_l2_block_pagetable 753 7541: 755#endif 756 757 /* Link the l1 -> l2 table */ 758 mov x9, x6 759 adrp x6, pagetable_l1_ttbr0_bootstrap 760 add x6, x6, :lo12:pagetable_l1_ttbr0_bootstrap 761 bl link_l1_pagetable 762 763 /* Link the l0 -> l1 table */ 764 mov x9, x6 765 adrp x6, pagetable_l0_ttbr0_bootstrap 766 add x6, x6, :lo12:pagetable_l0_ttbr0_bootstrap 767 mov x10, #1 768 bl link_l0_pagetable 769 770 /* Save the TTBR0 table physical address */ 771 mov x27, x6 772 773 /* Restore the Link register */ 774 mov x30, x5 775 ret 776LEND(create_pagetables) 777 778/* 779 * Builds an L0 -> L1 table descriptor 780 * 781 * x6 = L0 table 782 * x8 = Virtual Address 783 * x9 = L1 PA (trashed) 784 * x10 = Entry count (trashed) 785 * x11, x12 and x13 are trashed 786 */ 787LENTRY(link_l0_pagetable) 788 /* 789 * Link an L0 -> L1 table entry. 790 */ 791 /* Find the table index */ 792 lsr x11, x8, #L0_SHIFT 793 and x11, x11, #L0_ADDR_MASK 794 795 /* Build the L0 block entry */ 796 mov x12, #L0_TABLE 797 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 798 799 /* Only use the output address bits */ 800 lsr x9, x9, #PAGE_SHIFT 8011: orr x13, x12, x9, lsl #PAGE_SHIFT 802 803 /* Store the entry */ 804 str x13, [x6, x11, lsl #3] 805 806 sub x10, x10, #1 807 add x11, x11, #1 808 add x9, x9, #1 809 cbnz x10, 1b 810 811 ret 812LEND(link_l0_pagetable) 813 814/* 815 * Builds an L1 -> L2 table descriptor 816 * 817 * x6 = L1 table 818 * x8 = Virtual Address 819 * x9 = L2 PA (trashed) 820 * x11, x12 and x13 are trashed 821 */ 822LENTRY(link_l1_pagetable) 823 /* 824 * Link an L1 -> L2 table entry. 825 */ 826 /* Find the table index */ 827 lsr x11, x8, #L1_SHIFT 828 and x11, x11, #Ln_ADDR_MASK 829 830 /* Build the L1 block entry */ 831 mov x12, #L1_TABLE 832 833 /* Only use the output address bits */ 834 lsr x9, x9, #PAGE_SHIFT 835 orr x13, x12, x9, lsl #PAGE_SHIFT 836 837 /* Store the entry */ 838 str x13, [x6, x11, lsl #3] 839 840 ret 841LEND(link_l1_pagetable) 842 843/* 844 * Builds count 2 MiB page table entry 845 * x6 = L2 table 846 * x7 = Block attributes 847 * x8 = VA start 848 * x9 = PA start (trashed) 849 * x10 = Entry count (trashed) 850 * x11, x12 and x13 are trashed 851 */ 852LENTRY(build_l2_block_pagetable) 853 /* 854 * Build the L2 table entry. 855 */ 856 /* Find the table index */ 857 lsr x11, x8, #L2_SHIFT 858 and x11, x11, #Ln_ADDR_MASK 859 860 /* Build the L2 block entry */ 861 orr x12, x7, #L2_BLOCK 862 orr x12, x12, #(ATTR_AF) 863 orr x12, x12, #(ATTR_S1_UXN) 864#ifdef __ARM_FEATURE_BTI_DEFAULT 865 orr x12, x12, x21 866#endif 867 /* Set the shareability attribute */ 868 orr x12, x12, x22 869 870 /* Only use the output address bits */ 871 lsr x9, x9, #L2_SHIFT 872 873 /* Set the physical address for this virtual address */ 8741: orr x13, x12, x9, lsl #L2_SHIFT 875 876 /* Store the entry */ 877 str x13, [x6, x11, lsl #3] 878 879 sub x10, x10, #1 880 add x11, x11, #1 881 add x9, x9, #1 882 cbnz x10, 1b 883 884 ret 885LEND(build_l2_block_pagetable) 886 887/* 888 * Builds an L2 -> L3 table descriptor 889 * 890 * x6 = L2 table 891 * x8 = Virtual Address 892 * x9 = L3 PA (trashed) 893 * x11, x12 and x13 are trashed 894 */ 895LENTRY(link_l2_pagetable) 896 /* 897 * Link an L2 -> L3 table entry. 898 */ 899 /* Find the table index */ 900 lsr x11, x8, #L2_SHIFT 901 and x11, x11, #Ln_ADDR_MASK 902 903 /* Build the L1 block entry */ 904 mov x12, #L2_TABLE 905 906 /* Only use the output address bits */ 907 lsr x9, x9, #PAGE_SHIFT 908 orr x13, x12, x9, lsl #PAGE_SHIFT 909 910 /* Store the entry */ 911 str x13, [x6, x11, lsl #3] 912 913 ret 914LEND(link_l2_pagetable) 915 916/* 917 * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create 918 * large page (L3C) mappings when the current VA and remaining count allow 919 * it. 920 * x6 = L3 table 921 * x7 = Block attributes 922 * x8 = VA start 923 * x9 = PA start (trashed) 924 * x10 = Entry count (trashed) 925 * x11, x12 and x13 are trashed 926 * 927 * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. 928 */ 929LENTRY(build_l3_page_pagetable) 930 cbz x10, 2f 931 /* 932 * Build the L3 table entry. 933 */ 934 /* Find the table index */ 935 lsr x11, x8, #L3_SHIFT 936 and x11, x11, #Ln_ADDR_MASK 937 938 /* Build the L3 page entry */ 939 orr x12, x7, #L3_PAGE 940 orr x12, x12, #(ATTR_AF) 941 orr x12, x12, #(ATTR_S1_UXN) 942#ifdef __ARM_FEATURE_BTI_DEFAULT 943 orr x12, x12, x21 944#endif 945 /* Set the shareability attribute */ 946 orr x12, x12, x22 947 948 /* Only use the output address bits */ 949 lsr x9, x9, #L3_SHIFT 950 951 /* Check if an ATTR_CONTIGUOUS mapping is possible */ 9521: tst x11, #(L3C_ENTRIES - 1) 953 b.ne 2f 954 cmp x10, #L3C_ENTRIES 955 b.lo 3f 956 orr x12, x12, #(ATTR_CONTIGUOUS) 957 b 2f 9583: and x12, x12, #(~ATTR_CONTIGUOUS) 959 960 /* Set the physical address for this virtual address */ 9612: orr x13, x12, x9, lsl #L3_SHIFT 962 963 /* Store the entry */ 964 str x13, [x6, x11, lsl #3] 965 966 sub x10, x10, #1 967 add x11, x11, #1 968 add x9, x9, #1 969 cbnz x10, 1b 9702: 971 972 ret 973LEND(build_l3_page_pagetable) 974 975LENTRY(start_mmu) 976 dsb sy 977 978 /* Load the exception vectors */ 979 ldr x2, =exception_vectors 980 msr vbar_el1, x2 981 982 /* Load ttbr0 and ttbr1 */ 983 msr ttbr0_el1, x27 984 msr ttbr1_el1, x24 985 isb 986 987 /* Clear the Monitor Debug System control register */ 988 msr mdscr_el1, xzr 989 990 /* Invalidate the TLB */ 991 tlbi vmalle1is 992 dsb ish 993 isb 994 995 ldr x2, mair 996 msr mair_el1, x2 997 998 /* 999 * Setup TCR according to the PARange and ASIDBits fields 1000 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 1001 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 1002 * to 1 only if the ASIDBits field equals 0b0010. 1003 */ 1004 ldr x2, tcr 1005 1006 /* If x22 contains a non-zero value then LPA2 is not implemented */ 1007 cbnz x22, .Lno_lpa2 1008 ldr x3, =(TCR_DS) 1009 orr x2, x2, x3 1010.Lno_lpa2: 1011 1012 mrs x3, id_aa64mmfr0_el1 1013 1014 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 1015 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 1016 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 1017 1018 /* Check if the HW supports 16 bit ASIDS */ 1019 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 1020 /* If so x3 == 1, else x3 == 0 */ 1021 cset x3, eq 1022 /* Set TCR.AS with x3 */ 1023 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 1024 1025 /* 1026 * Check if the HW supports access flag updates, and set 1027 * TCR_EL1.HA accordingly. The TCR_EL1.HD flag to enable 1028 * HW management of dirty state is set in C code as it may 1029 * need to be disabled because of CPU errata. 1030 */ 1031 CHECK_CPU_FEAT(x3, ID_AA64MMFR1, HAFDBS, AF, 1f) 1032 orr x2, x2, #(TCR_HA) 10331: 1034 1035 msr tcr_el1, x2 1036 1037 /* 1038 * Setup SCTLR. 1039 */ 1040 ldr x2, sctlr_set 1041 ldr x3, sctlr_clear 1042 mrs x1, sctlr_el1 1043 bic x1, x1, x3 /* Clear the required bits */ 1044 orr x1, x1, x2 /* Set the required bits */ 1045 msr sctlr_el1, x1 1046 isb 1047 1048 ret 1049 1050 .align 3 1051mair: 1052 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 1053 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 1054 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 1055 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 1056 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 1057tcr: 1058#if PAGE_SIZE == PAGE_SIZE_4K 1059#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 1060#elif PAGE_SIZE == PAGE_SIZE_16K 1061#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 1062#else 1063#error Unsupported page size 1064#endif 1065 1066 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 1067 TCR_SH1_IS | TCR_ORGN1_WBWA | TCR_IRGN1_WBWA | \ 1068 TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) 1069sctlr_set: 1070 /* Bits to set */ 1071 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 1072 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 1073 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 1074 SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 1075sctlr_clear: 1076 /* Bits to clear */ 1077 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 1078 SCTLR_ITD | SCTLR_A) 1079LEND(start_mmu) 1080 1081ENTRY(abort) 1082 b abort 1083END(abort) 1084 1085.bss 1086 .align PAGE_SHIFT 1087initstack: 1088 .space BOOT_STACK_SIZE 1089initstack_end: 1090 1091 .section .init_pagetable, "aw", %nobits 1092 .align PAGE_SHIFT 1093 /* 1094 * 6 initial tables (in the following order): 1095 * L2 for kernel (High addresses) 1096 * L1 for kernel 1097 * L0 for kernel 1098 * L1 bootstrap for user (Low addresses) 1099 * L0 bootstrap for user 1100 * L0 for user 1101 */ 1102 .globl pagetable_l0_ttbr1 1103pagetable: 1104pagetable_l3_ttbr1: 1105 .space (PAGE_SIZE * L3_PAGE_COUNT) 1106pagetable_l2_ttbr1: 1107 .space PAGE_SIZE 1108pagetable_l1_ttbr1: 1109 .space PAGE_SIZE 1110pagetable_l0_ttbr1: 1111 .space PAGE_SIZE 1112pagetable_l2_ttbr0_bootstrap: 1113 .space PAGE_SIZE 1114pagetable_l1_ttbr0_bootstrap: 1115 .space PAGE_SIZE 1116pagetable_l0_ttbr0_bootstrap: 1117 .space PAGE_SIZE 1118pagetable_l0_ttbr0: 1119 .space PAGE_SIZE 1120pagetable_end: 1121 1122el2_pagetable: 1123 .space PAGE_SIZE 1124 1125 .section .rodata, "a", %progbits 1126 .globl aarch32_sigcode 1127 .align 2 1128aarch32_sigcode: 1129 .word 0xe1a0000d // mov r0, sp 1130 .word 0xe2800040 // add r0, r0, #SIGF_UC 1131 .word 0xe59f700c // ldr r7, [pc, #12] 1132 .word 0xef000000 // swi #0 1133 .word 0xe59f7008 // ldr r7, [pc, #8] 1134 .word 0xef000000 // swi #0 1135 .word 0xeafffffa // b . - 16 1136 .word SYS_sigreturn 1137 .word SYS_exit 1138 .align 3 1139 .size aarch32_sigcode, . - aarch32_sigcode 1140aarch32_esigcode: 1141 .data 1142 .global sz_aarch32_sigcode 1143sz_aarch32_sigcode: 1144 .quad aarch32_esigcode - aarch32_sigcode 1145 1146GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 1147