1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/elf_common.h> 30#include <sys/syscall.h> 31#include <machine/asm.h> 32#include <machine/armreg.h> 33#include <machine/cpu.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42#if PAGE_SIZE == PAGE_SIZE_16K 43/* 44 * The number of level 3 tables to create. 32 will allow for 1G of address 45 * space, the same as a single level 2 page with 4k pages. 46 */ 47#define L3_PAGE_COUNT 32 48#endif 49 50/* 51 * The size of our bootstrap stack. 52 */ 53#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 54 55 .globl kernbase 56 .set kernbase, KERNBASE 57 58/* 59 * We assume: 60 * MMU on with an identity map, or off 61 * D-Cache: off 62 * I-Cache: on or off 63 * We are loaded at a 2MiB aligned address 64 */ 65 66ENTRY(_start) 67 /* Enter the kernel exception level */ 68 bl enter_kernel_el 69 70 /* Set the context id */ 71 msr contextidr_el1, xzr 72 73 /* Get the virt -> phys offset */ 74 bl get_load_phys_addr 75 76 /* 77 * At this point: 78 * x28 = Our physical load address 79 */ 80 81 /* Create the page tables */ 82 bl create_pagetables 83 84 /* 85 * At this point: 86 * x27 = TTBR0 table 87 * x26 = Kernel L1 table 88 * x24 = TTBR1 table 89 * x22 = PTE shareability attributes 90 */ 91 92 /* Enable the mmu */ 93 bl start_mmu 94 95 /* Load the new ttbr0 pagetable */ 96 adrp x27, pagetable_l0_ttbr0 97 add x27, x27, :lo12:pagetable_l0_ttbr0 98 99 /* Jump to the virtual address space */ 100 ldr x15, .Lvirtdone 101 br x15 102 103virtdone: 104 BTI_J 105 106 /* Set up the stack */ 107 adrp x25, initstack_end 108 add x25, x25, :lo12:initstack_end 109 sub sp, x25, #PCB_SIZE 110 111 /* Zero the BSS */ 112 ldr x15, .Lbss 113 ldr x14, .Lend 1141: 115 stp xzr, xzr, [x15], #16 116 cmp x15, x14 117 b.lo 1b 118 119#if defined(PERTHREAD_SSP) 120 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 121 adrp x15, boot_canary 122 add x15, x15, :lo12:boot_canary 123 msr sp_el0, x15 124#endif 125 126 /* Backup the module pointer */ 127 mov x1, x0 128 129 sub sp, sp, #BOOTPARAMS_SIZE 130 mov x0, sp 131 132 str x1, [x0, #BP_MODULEP] 133 adrp x25, initstack 134 add x25, x25, :lo12:initstack 135 str x25, [x0, #BP_KERN_STACK] 136 str x27, [x0, #BP_KERN_TTBR0] 137 str x23, [x0, #BP_BOOT_EL] 138 139 /* Set this before it's used in kasan_init_early */ 140 adrp x1, pmap_sh_attr 141 str x22, [x1, :lo12:pmap_sh_attr] 142 143#ifdef KASAN 144 /* Save bootparams */ 145 mov x19, x0 146 147 /* Bootstrap an early shadow map for the boot stack. */ 148 ldr x0, [x0, #BP_KERN_STACK] 149 ldr x1, =BOOT_STACK_SIZE 150 bl kasan_init_early 151 152 /* Restore bootparams */ 153 mov x0, x19 154#endif 155 156 /* trace back starts here */ 157 mov fp, #0 158 /* Branch to C code */ 159 bl initarm 160 /* We are done with the boot params */ 161 add sp, sp, #BOOTPARAMS_SIZE 162 163 /* 164 * Enable pointer authentication in the kernel. We set the keys for 165 * thread0 in initarm so have to wait until it returns to enable it. 166 * If we were to enable it in initarm then any authentication when 167 * returning would fail as it was called with pointer authentication 168 * disabled. 169 */ 170 bl ptrauth_start 171 172 bl mi_startup 173 174 /* We should not get here */ 175 brk 0 176 177 .align 3 178.Lvirtdone: 179 .quad virtdone 180.Lbss: 181 .quad __bss_start 182.Lend: 183 .quad __bss_end 184END(_start) 185 186#ifdef SMP 187/* 188 * void 189 * mpentry_psci(unsigned long) 190 * 191 * Called by a core when it is being brought online with psci. 192 * The data in x0 is passed straight to init_secondary. 193 */ 194ENTRY(mpentry_psci) 195 mov x26, xzr 196 b mpentry_common 197END(mpentry_psci) 198 199/* 200 * void 201 * mpentry_spintable(void) 202 * 203 * Called by a core when it is being brought online with a spin-table. 204 * Reads the new CPU ID and passes this to init_secondary. 205 */ 206ENTRY(mpentry_spintable) 207 ldr x26, =spintable_wait 208 b mpentry_common 209END(mpentry_spintable) 210 211/* Wait for the current CPU to be released */ 212LENTRY(spintable_wait) 213 /* Read the affinity bits from mpidr_el1 */ 214 mrs x1, mpidr_el1 215 ldr x2, =CPU_AFF_MASK 216 and x1, x1, x2 217 218 adrp x2, ap_cpuid 2191: 220 ldr x0, [x2, :lo12:ap_cpuid] 221 cmp x0, x1 222 b.ne 1b 223 224 str xzr, [x2, :lo12:ap_cpuid] 225 dsb sy 226 sev 227 228 ret 229LEND(mpentry_spintable) 230 231LENTRY(mpentry_common) 232 /* Disable interrupts */ 233 msr daifset, #DAIF_INTR 234 235 /* Enter the kernel exception level */ 236 bl enter_kernel_el 237 238 /* Set the context id */ 239 msr contextidr_el1, xzr 240 241 /* Load the kernel page table */ 242 adrp x24, pagetable_l0_ttbr1 243 add x24, x24, :lo12:pagetable_l0_ttbr1 244 /* Load the identity page table */ 245 adrp x27, pagetable_l0_ttbr0_bootstrap 246 add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 247 248 /* Enable the mmu */ 249 bl start_mmu 250 251 /* Load the new ttbr0 pagetable */ 252 adrp x27, pagetable_l0_ttbr0 253 add x27, x27, :lo12:pagetable_l0_ttbr0 254 255 /* Jump to the virtual address space */ 256 ldr x15, =mp_virtdone 257 br x15 258 259mp_virtdone: 260 BTI_J 261 262 /* 263 * Allow this CPU to wait until the kernel is ready for it, 264 * e.g. with spin-table but each CPU uses the same release address 265 */ 266 cbz x26, 1f 267 blr x26 2681: 269 270 /* Start using the AP boot stack */ 271 adrp x4, bootstack 272 ldr x4, [x4, :lo12:bootstack] 273 mov sp, x4 274 275#if defined(PERTHREAD_SSP) 276 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 277 adrp x15, boot_canary 278 add x15, x15, :lo12:boot_canary 279 msr sp_el0, x15 280#endif 281 282 /* Load the kernel ttbr0 pagetable */ 283 msr ttbr0_el1, x27 284 isb 285 286 /* Invalidate the TLB */ 287 tlbi vmalle1 288 dsb sy 289 isb 290 291 /* 292 * Initialize the per-CPU pointer before calling into C code, for the 293 * benefit of kernel sanitizers. 294 */ 295 adrp x18, bootpcpu 296 ldr x18, [x18, :lo12:bootpcpu] 297 msr tpidr_el1, x18 298 299 b init_secondary 300LEND(mpentry_common) 301#endif 302 303/* 304 * Enter the exception level the kernel will use: 305 * 306 * - If in EL1 continue in EL1 307 * - If the CPU supports FEAT_VHE then set HCR_E2H and HCR_TGE and continue 308 * in EL2 309 * - Configure EL2 to support running the kernel at EL1 and exit to that 310 */ 311LENTRY(enter_kernel_el) 312#define INIT_SCTLR_EL1 (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_EIS | \ 313 SCTLR_TSCXT | SCTLR_EOS) 314 mrs x23, CurrentEL 315 and x23, x23, #(CURRENTEL_EL_MASK) 316 cmp x23, #(CURRENTEL_EL_EL2) 317 b.eq 1f 318 319 ldr x2, =INIT_SCTLR_EL1 320 msr sctlr_el1, x2 321 /* SCTLR_EOS is set so eret is a context synchronizing event so we 322 * need an isb here to ensure it's observed by later instructions, 323 * but don't need it in the eret below. 324 */ 325 isb 326 327 /* Ensure SPSR_EL1 and pstate are in sync. The only wat to set the 328 * latter is to set the former and return from an exception with eret. 329 */ 330 mov x2, #(PSR_DAIF | PSR_M_EL1h) 331 msr spsr_el1, x2 332 msr elr_el1, lr 333 eret 334 3351: 336 dsb sy 337 /* 338 * Set just the reserved bits in sctlr_el2. This will disable the 339 * MMU which may have broken the kernel if we enter the kernel in 340 * EL2, e.g. when using VHE. 341 */ 342 ldr x2, =(SCTLR_EL2_RES1 | SCTLR_EL2_EIS | SCTLR_EL2_EOS) 343 msr sctlr_el2, x2 344 isb 345 346 /* Configure the Hypervisor */ 347 ldr x2, =(HCR_RW | HCR_APK | HCR_API | HCR_E2H) 348 msr hcr_el2, x2 349 350 /* Stash value of HCR_EL2 for later */ 351 isb 352 mrs x4, hcr_el2 353 354 /* Load the Virtualization Process ID Register */ 355 mrs x2, midr_el1 356 msr vpidr_el2, x2 357 358 /* Load the Virtualization Multiprocess ID Register */ 359 mrs x2, mpidr_el1 360 msr vmpidr_el2, x2 361 362 /* Set the initial sctlr_el1 */ 363 ldr x2, =INIT_SCTLR_EL1 364 msr sctlr_el1, x2 365 366 /* Check if the E2H flag is set */ 367 tst x4, #HCR_E2H 368 b.eq .Lno_vhe 369 370 /* 371 * The kernel will be running in EL2, route exceptions here rather 372 * than EL1. 373 */ 374 orr x4, x4, #(HCR_TGE) 375 msr hcr_el2, x4 376 isb 377 378 msr SCTLR_EL12_REG, x2 379 mov x2, xzr /* CPTR_EL2 is managed by vfp.c */ 380 ldr x3, =(CNTHCTL_E2H_EL1PCTEN | CNTHCTL_E2H_EL1PTEN) 381 ldr x5, =(PSR_DAIF | PSR_M_EL2h) 382 b .Ldone_vhe 383 384.Lno_vhe: 385 /* Hypervisor trap functions */ 386 adrp x2, hyp_stub_vectors 387 add x2, x2, :lo12:hyp_stub_vectors 388 msr vbar_el2, x2 389 390 ldr x2, =(CPTR_RES1) 391 ldr x3, =(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 392 ldr x5, =(PSR_DAIF | PSR_M_EL1h) 393 394.Ldone_vhe: 395 396 msr cptr_el2, x2 397 /* Enable access to the physical timers at EL1 */ 398 msr cnthctl_el2, x3 399 /* Set the return PSTATE */ 400 msr spsr_el2, x5 401 402 /* 403 * Configure the Extended Hypervisor register. This is only valid if 404 * FEAT_HCX is enabled. 405 */ 406 mrs x2, id_aa64mmfr1_el1 407 ubfx x2, x2, #ID_AA64MMFR1_HCX_SHIFT, #ID_AA64MMFR1_HCX_WIDTH 408 cbz x2, 2f 409 410 /* Extended Hypervisor Configuration */ 411 mov x2, xzr 412 msr HCRX_EL2_REG, x2 413 isb 4142: 415 416 /* Don't trap to EL2 for CP15 traps */ 417 msr hstr_el2, xzr 418 419 /* Set the counter offset to a known value */ 420 msr cntvoff_el2, xzr 421 422 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 423 msr vttbr_el2, xzr 424 425 /* Configure GICv3 CPU interface */ 426 mrs x2, id_aa64pfr0_el1 427 /* Extract GIC bits from the register */ 428 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 429 /* GIC[3:0] != 0000 - GIC CPU interface via special regs. supported */ 430 cbz x2, 3f 431 432 mrs x2, icc_sre_el2 433 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 434 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 435 msr icc_sre_el2, x2 4363: 437 438 /* Set the address to return to our return address */ 439 msr elr_el2, x30 440 isb 441 442 eret 443#undef INIT_SCTLR_EL1 444LEND(enter_kernel_el) 445 446/* 447 * Get the physical address the kernel was loaded at. 448 */ 449LENTRY(get_load_phys_addr) 450 /* Load the offset of get_load_phys_addr from KERNBASE */ 451 ldr x28, =(get_load_phys_addr - KERNBASE) 452 /* Load the physical address of get_load_phys_addr */ 453 adr x29, get_load_phys_addr 454 /* Find the physical address of KERNBASE, i.e. our load address */ 455 sub x28, x29, x28 456 ret 457LEND(get_load_phys_addr) 458 459/* 460 * This builds the page tables containing the identity map, and the kernel 461 * virtual map. 462 * 463 * It relys on: 464 * We were loaded to an address that is on a 2MiB boundary 465 * All the memory must not cross a 1GiB boundaty 466 * x28 contains the physical address we were loaded from 467 * 468 * There are 7 or 8 pages before that address for the page tables 469 * The pages used are: 470 * - The Kernel L3 tables (only for 16k kernel) 471 * - The Kernel L2 table 472 * - The Kernel L1 table 473 * - The Kernel L0 table (TTBR1) 474 * - The identity (PA = VA) L2 table 475 * - The identity (PA = VA) L1 table 476 * - The identity (PA = VA) L0 table (Early TTBR0) 477 * - The Kernel empty L0 table (Late TTBR0) 478 */ 479LENTRY(create_pagetables) 480 /* Save the Link register */ 481 mov x5, x30 482 483 /* Clean the page table */ 484 adrp x6, pagetable 485 add x6, x6, :lo12:pagetable 486 mov x26, x6 487 adrp x27, pagetable_end 488 add x27, x27, :lo12:pagetable_end 4891: 490 stp xzr, xzr, [x6], #16 491 stp xzr, xzr, [x6], #16 492 stp xzr, xzr, [x6], #16 493 stp xzr, xzr, [x6], #16 494 cmp x6, x27 495 b.lo 1b 496 497 /* 498 * Find the shareability attribute we should use. If FEAT_LPA2 is 499 * enabled then the shareability field is moved from the page table 500 * to tcr_el1 and the bits in the page table are reused by the 501 * address field. 502 */ 503#if PAGE_SIZE == PAGE_SIZE_4K 504#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK 505#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2 506#elif PAGE_SIZE == PAGE_SIZE_16K 507#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK 508#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2 509#else 510#error Unsupported page size 511#endif 512 mrs x6, id_aa64mmfr0_el1 513 mov x7, LPA2_VAL 514 and x6, x6, LPA2_MASK 515 cmp x6, x7 516 ldr x22, =(ATTR_SH(ATTR_SH_IS)) 517 csel x22, xzr, x22, eq 518#undef LPA2_MASK 519#undef LPA2_VAL 520 521 /* 522 * Build the TTBR1 maps. 523 */ 524 525 /* Find the size of the kernel */ 526 mov x6, #(KERNBASE) 527 528#if defined(LINUX_BOOT_ABI) 529 /* X19 is used as 'map FDT data' flag */ 530 mov x19, xzr 531 532 /* No modules or FDT pointer ? */ 533 cbz x0, booti_no_fdt 534 535 /* 536 * Test if x0 points to modules descriptor(virtual address) or 537 * to FDT (physical address) 538 */ 539 cmp x0, x6 /* x6 is #(KERNBASE) */ 540 b.lo booti_fdt 541#endif 542 543 /* Booted with modules pointer */ 544 /* Find modulep - begin */ 545 sub x8, x0, x6 546 /* 547 * Add space for the module data. When PAGE_SIZE is 4k this will 548 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 549 * larger it will be at least as large as we use smaller level 3 550 * pages. 551 */ 552 ldr x7, =((6 * 1024 * 1024) - 1) 553 add x8, x8, x7 554 b common 555 556#if defined(LINUX_BOOT_ABI) 557booti_fdt: 558 /* Booted by U-Boot booti with FDT data */ 559 /* Set 'map FDT data' flag */ 560 mov x19, #1 561 562booti_no_fdt: 563 /* Booted by U-Boot booti without FTD data */ 564 /* Find the end - begin */ 565 ldr x7, .Lend 566 sub x8, x7, x6 567 568 /* 569 * Add one 2MiB page for copy of FDT data (maximum FDT size), 570 * one for metadata and round up 571 */ 572 ldr x7, =(3 * L2_SIZE - 1) 573 add x8, x8, x7 574#endif 575 576common: 577#if PAGE_SIZE != PAGE_SIZE_4K 578 /* 579 * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned 580 * address, enabling the creation of L3C pages. However, when the page 581 * size is larger than 4k, L2 blocks are too large to map the kernel 582 * with 2M alignment. 583 */ 584#define PTE_SHIFT L3_SHIFT 585#define BUILD_PTE_FUNC build_l3_page_pagetable 586#else 587#define PTE_SHIFT L2_SHIFT 588#define BUILD_PTE_FUNC build_l2_block_pagetable 589#endif 590 591 /* Get the number of blocks/pages to allocate, rounded down */ 592 lsr x10, x8, #(PTE_SHIFT) 593 594 /* Create the kernel space PTE table */ 595 mov x6, x26 596 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 597 mov x8, #(KERNBASE) 598 mov x9, x28 599 bl BUILD_PTE_FUNC 600 601#undef PTE_SHIFT 602#undef BUILD_PTE_FUNC 603 604#if PAGE_SIZE != PAGE_SIZE_4K 605 /* Move to the l2 table */ 606 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 607 add x26, x26, x9 608 609 /* Link the l2 -> l3 table */ 610 mov x9, x6 611 mov x6, x26 612 bl link_l2_pagetable 613#endif 614 615 /* Move to the l1 table */ 616 add x26, x26, #PAGE_SIZE 617 618 /* Link the l1 -> l2 table */ 619 mov x9, x6 620 mov x6, x26 621 bl link_l1_pagetable 622 623 /* Move to the l0 table */ 624 add x24, x26, #PAGE_SIZE 625 626 /* Link the l0 -> l1 table */ 627 mov x9, x6 628 mov x6, x24 629 mov x10, #1 630 bl link_l0_pagetable 631 632 /* 633 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 634 * They are only needed early on, so the VA = PA map is uncached. 635 */ 636 add x27, x24, #PAGE_SIZE 637 638 mov x6, x27 /* The initial page table */ 639 640 /* Create the VA = PA map */ 641 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 642 adrp x16, _start 643 and x16, x16, #(~L2_OFFSET) 644 mov x9, x16 /* PA start */ 645 mov x8, x16 /* VA start (== PA start) */ 646 mov x10, #1 647 bl build_l2_block_pagetable 648 649#if defined(SOCDEV_PA) 650 /* Create a table for the UART */ 651 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 652 ldr x9, =(L2_SIZE) 653 add x16, x16, x9 /* VA start */ 654 mov x8, x16 655 656 /* Store the socdev virtual address */ 657 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 658 adrp x9, socdev_va 659 str x17, [x9, :lo12:socdev_va] 660 661 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 662 mov x10, #1 663 bl build_l2_block_pagetable 664#endif 665 666#if defined(LINUX_BOOT_ABI) 667 /* Map FDT data ? */ 668 cbz x19, 1f 669 670 /* Create the mapping for FDT data (2 MiB max) */ 671 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 672 ldr x9, =(L2_SIZE) 673 add x16, x16, x9 /* VA start */ 674 mov x8, x16 675 mov x9, x0 /* PA start */ 676 /* Update the module pointer to point at the allocated memory */ 677 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 678 add x0, x0, x8 /* Add the aligned virtual address */ 679 680 mov x10, #1 681 bl build_l2_block_pagetable 682 6831: 684#endif 685 686 /* Move to the l1 table */ 687 add x27, x27, #PAGE_SIZE 688 689 /* Link the l1 -> l2 table */ 690 mov x9, x6 691 mov x6, x27 692 bl link_l1_pagetable 693 694 /* Move to the l0 table */ 695 add x27, x27, #PAGE_SIZE 696 697 /* Link the l0 -> l1 table */ 698 mov x9, x6 699 mov x6, x27 700 mov x10, #1 701 bl link_l0_pagetable 702 703 /* Restore the Link register */ 704 mov x30, x5 705 ret 706LEND(create_pagetables) 707 708/* 709 * Builds an L0 -> L1 table descriptor 710 * 711 * x6 = L0 table 712 * x8 = Virtual Address 713 * x9 = L1 PA (trashed) 714 * x10 = Entry count (trashed) 715 * x11, x12 and x13 are trashed 716 */ 717LENTRY(link_l0_pagetable) 718 /* 719 * Link an L0 -> L1 table entry. 720 */ 721 /* Find the table index */ 722 lsr x11, x8, #L0_SHIFT 723 and x11, x11, #L0_ADDR_MASK 724 725 /* Build the L0 block entry */ 726 mov x12, #L0_TABLE 727 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 728 729 /* Only use the output address bits */ 730 lsr x9, x9, #PAGE_SHIFT 7311: orr x13, x12, x9, lsl #PAGE_SHIFT 732 733 /* Store the entry */ 734 str x13, [x6, x11, lsl #3] 735 736 sub x10, x10, #1 737 add x11, x11, #1 738 add x9, x9, #1 739 cbnz x10, 1b 740 741 ret 742LEND(link_l0_pagetable) 743 744/* 745 * Builds an L1 -> L2 table descriptor 746 * 747 * x6 = L1 table 748 * x8 = Virtual Address 749 * x9 = L2 PA (trashed) 750 * x11, x12 and x13 are trashed 751 */ 752LENTRY(link_l1_pagetable) 753 /* 754 * Link an L1 -> L2 table entry. 755 */ 756 /* Find the table index */ 757 lsr x11, x8, #L1_SHIFT 758 and x11, x11, #Ln_ADDR_MASK 759 760 /* Build the L1 block entry */ 761 mov x12, #L1_TABLE 762 763 /* Only use the output address bits */ 764 lsr x9, x9, #PAGE_SHIFT 765 orr x13, x12, x9, lsl #PAGE_SHIFT 766 767 /* Store the entry */ 768 str x13, [x6, x11, lsl #3] 769 770 ret 771LEND(link_l1_pagetable) 772 773/* 774 * Builds count 2 MiB page table entry 775 * x6 = L2 table 776 * x7 = Block attributes 777 * x8 = VA start 778 * x9 = PA start (trashed) 779 * x10 = Entry count (trashed) 780 * x11, x12 and x13 are trashed 781 */ 782LENTRY(build_l2_block_pagetable) 783 /* 784 * Build the L2 table entry. 785 */ 786 /* Find the table index */ 787 lsr x11, x8, #L2_SHIFT 788 and x11, x11, #Ln_ADDR_MASK 789 790 /* Build the L2 block entry */ 791 orr x12, x7, #L2_BLOCK 792 orr x12, x12, #(ATTR_AF) 793 orr x12, x12, #(ATTR_S1_UXN) 794#ifdef __ARM_FEATURE_BTI_DEFAULT 795 orr x12, x12, #(ATTR_S1_GP) 796#endif 797 /* Set the shareability attribute */ 798 orr x12, x12, x22 799 800 /* Only use the output address bits */ 801 lsr x9, x9, #L2_SHIFT 802 803 /* Set the physical address for this virtual address */ 8041: orr x13, x12, x9, lsl #L2_SHIFT 805 806 /* Store the entry */ 807 str x13, [x6, x11, lsl #3] 808 809 sub x10, x10, #1 810 add x11, x11, #1 811 add x9, x9, #1 812 cbnz x10, 1b 813 814 ret 815LEND(build_l2_block_pagetable) 816 817#if PAGE_SIZE != PAGE_SIZE_4K 818/* 819 * Builds an L2 -> L3 table descriptor 820 * 821 * x6 = L2 table 822 * x8 = Virtual Address 823 * x9 = L3 PA (trashed) 824 * x11, x12 and x13 are trashed 825 */ 826LENTRY(link_l2_pagetable) 827 /* 828 * Link an L2 -> L3 table entry. 829 */ 830 /* Find the table index */ 831 lsr x11, x8, #L2_SHIFT 832 and x11, x11, #Ln_ADDR_MASK 833 834 /* Build the L1 block entry */ 835 mov x12, #L2_TABLE 836 837 /* Only use the output address bits */ 838 lsr x9, x9, #PAGE_SHIFT 839 orr x13, x12, x9, lsl #PAGE_SHIFT 840 841 /* Store the entry */ 842 str x13, [x6, x11, lsl #3] 843 844 ret 845LEND(link_l2_pagetable) 846 847/* 848 * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create 849 * large page (L3C) mappings when the current VA and remaining count allow 850 * it. 851 * x6 = L3 table 852 * x7 = Block attributes 853 * x8 = VA start 854 * x9 = PA start (trashed) 855 * x10 = Entry count (trashed) 856 * x11, x12 and x13 are trashed 857 * 858 * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. 859 */ 860LENTRY(build_l3_page_pagetable) 861 /* 862 * Build the L3 table entry. 863 */ 864 /* Find the table index */ 865 lsr x11, x8, #L3_SHIFT 866 and x11, x11, #Ln_ADDR_MASK 867 868 /* Build the L3 page entry */ 869 orr x12, x7, #L3_PAGE 870 orr x12, x12, #(ATTR_AF) 871 orr x12, x12, #(ATTR_S1_UXN) 872#ifdef __ARM_FEATURE_BTI_DEFAULT 873 orr x12, x12, #(ATTR_S1_GP) 874#endif 875 /* Set the shareability attribute */ 876 orr x12, x12, x22 877 878 /* Only use the output address bits */ 879 lsr x9, x9, #L3_SHIFT 880 881 /* Check if an ATTR_CONTIGUOUS mapping is possible */ 8821: tst x11, #(L3C_ENTRIES - 1) 883 b.ne 2f 884 cmp x10, #L3C_ENTRIES 885 b.lo 3f 886 orr x12, x12, #(ATTR_CONTIGUOUS) 887 b 2f 8883: and x12, x12, #(~ATTR_CONTIGUOUS) 889 890 /* Set the physical address for this virtual address */ 8912: orr x13, x12, x9, lsl #L3_SHIFT 892 893 /* Store the entry */ 894 str x13, [x6, x11, lsl #3] 895 896 sub x10, x10, #1 897 add x11, x11, #1 898 add x9, x9, #1 899 cbnz x10, 1b 900 901 ret 902LEND(build_l3_page_pagetable) 903#endif 904 905LENTRY(start_mmu) 906 dsb sy 907 908 /* Load the exception vectors */ 909 ldr x2, =exception_vectors 910 msr vbar_el1, x2 911 912 /* Load ttbr0 and ttbr1 */ 913 msr ttbr0_el1, x27 914 msr ttbr1_el1, x24 915 isb 916 917 /* Clear the Monitor Debug System control register */ 918 msr mdscr_el1, xzr 919 920 /* Invalidate the TLB */ 921 tlbi vmalle1is 922 dsb ish 923 isb 924 925 ldr x2, mair 926 msr mair_el1, x2 927 928 /* 929 * Setup TCR according to the PARange and ASIDBits fields 930 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 931 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 932 * to 1 only if the ASIDBits field equals 0b0010. 933 */ 934 ldr x2, tcr 935 936 /* If x22 contains a non-zero value then LPA2 is not implemented */ 937 cbnz x22, .Lno_lpa2 938 ldr x3, =(TCR_DS) 939 orr x2, x2, x3 940.Lno_lpa2: 941 942 mrs x3, id_aa64mmfr0_el1 943 944 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 945 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 946 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 947 948 /* Check if the HW supports 16 bit ASIDS */ 949 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 950 /* If so x3 == 1, else x3 == 0 */ 951 cset x3, eq 952 /* Set TCR.AS with x3 */ 953 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 954 955 /* 956 * Check if the HW supports access flag updates, and set 957 * TCR_EL1.HA accordingly. The TCR_EL1.HD flag to enable 958 * HW management of dirty state is set in C code as it may 959 * need to be disabled because of CPU errata. 960 */ 961 mrs x3, id_aa64mmfr1_el1 962 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 963 cbz x3, 1f 964 orr x2, x2, #(TCR_HA) 9651: 966 967 msr tcr_el1, x2 968 969 /* 970 * Setup SCTLR. 971 */ 972 ldr x2, sctlr_set 973 ldr x3, sctlr_clear 974 mrs x1, sctlr_el1 975 bic x1, x1, x3 /* Clear the required bits */ 976 orr x1, x1, x2 /* Set the required bits */ 977 msr sctlr_el1, x1 978 isb 979 980 ret 981 982 .align 3 983mair: 984 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 985 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 986 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 987 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 988 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 989tcr: 990#if PAGE_SIZE == PAGE_SIZE_4K 991#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 992#elif PAGE_SIZE == PAGE_SIZE_16K 993#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 994#else 995#error Unsupported page size 996#endif 997 998 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 999 TCR_SH1_IS | TCR_ORGN1_WBWA | TCR_IRGN1_WBWA | \ 1000 TCR_SH0_IS | TCR_ORGN0_WBWA | TCR_IRGN0_WBWA) 1001sctlr_set: 1002 /* Bits to set */ 1003 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 1004 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 1005 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 1006 SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 1007sctlr_clear: 1008 /* Bits to clear */ 1009 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 1010 SCTLR_ITD | SCTLR_A) 1011LEND(start_mmu) 1012 1013ENTRY(abort) 1014 b abort 1015END(abort) 1016 1017.bss 1018 .align PAGE_SHIFT 1019initstack: 1020 .space BOOT_STACK_SIZE 1021initstack_end: 1022 1023 .section .init_pagetable, "aw", %nobits 1024 .align PAGE_SHIFT 1025 /* 1026 * 6 initial tables (in the following order): 1027 * L2 for kernel (High addresses) 1028 * L1 for kernel 1029 * L0 for kernel 1030 * L1 bootstrap for user (Low addresses) 1031 * L0 bootstrap for user 1032 * L0 for user 1033 */ 1034 .globl pagetable_l0_ttbr1 1035pagetable: 1036#if PAGE_SIZE != PAGE_SIZE_4K 1037 .space (PAGE_SIZE * L3_PAGE_COUNT) 1038pagetable_l2_ttbr1: 1039#endif 1040 .space PAGE_SIZE 1041pagetable_l1_ttbr1: 1042 .space PAGE_SIZE 1043pagetable_l0_ttbr1: 1044 .space PAGE_SIZE 1045pagetable_l2_ttbr0_bootstrap: 1046 .space PAGE_SIZE 1047pagetable_l1_ttbr0_bootstrap: 1048 .space PAGE_SIZE 1049pagetable_l0_ttbr0_bootstrap: 1050 .space PAGE_SIZE 1051pagetable_l0_ttbr0: 1052 .space PAGE_SIZE 1053pagetable_end: 1054 1055el2_pagetable: 1056 .space PAGE_SIZE 1057 1058 .section .rodata, "a", %progbits 1059 .globl aarch32_sigcode 1060 .align 2 1061aarch32_sigcode: 1062 .word 0xe1a0000d // mov r0, sp 1063 .word 0xe2800040 // add r0, r0, #SIGF_UC 1064 .word 0xe59f700c // ldr r7, [pc, #12] 1065 .word 0xef000000 // swi #0 1066 .word 0xe59f7008 // ldr r7, [pc, #8] 1067 .word 0xef000000 // swi #0 1068 .word 0xeafffffa // b . - 16 1069 .word SYS_sigreturn 1070 .word SYS_exit 1071 .align 3 1072 .size aarch32_sigcode, . - aarch32_sigcode 1073aarch32_esigcode: 1074 .data 1075 .global sz_aarch32_sigcode 1076sz_aarch32_sigcode: 1077 .quad aarch32_esigcode - aarch32_sigcode 1078 1079GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) 1080