1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/syscall.h> 30#include <machine/asm.h> 31#include <machine/armreg.h> 32#include <machine/hypervisor.h> 33#include <machine/param.h> 34#include <machine/pte.h> 35#include <machine/vm.h> 36#include <machine/vmparam.h> 37 38#define VIRT_BITS 48 39 40#if PAGE_SIZE == PAGE_SIZE_16K 41/* 42 * The number of level 3 tables to create. 32 will allow for 1G of address 43 * space, the same as a single level 2 page with 4k pages. 44 */ 45#define L3_PAGE_COUNT 32 46#endif 47 48 .globl kernbase 49 .set kernbase, KERNBASE 50 51/* 52 * We assume: 53 * MMU on with an identity map, or off 54 * D-Cache: off 55 * I-Cache: on or off 56 * We are loaded at a 2MiB aligned address 57 */ 58 59ENTRY(_start) 60 /* Drop to EL1 */ 61 bl drop_to_el1 62 63 /* 64 * Disable the MMU. We may have entered the kernel with it on and 65 * will need to update the tables later. If this has been set up 66 * with anything other than a VA == PA map then this will fail, 67 * but in this case the code to find where we are running from 68 * would have also failed. 69 */ 70 dsb sy 71 mrs x2, sctlr_el1 72 bic x2, x2, SCTLR_M 73 msr sctlr_el1, x2 74 isb 75 76 /* Set the context id */ 77 msr contextidr_el1, xzr 78 79 /* Get the virt -> phys offset */ 80 bl get_load_phys_addr 81 82 /* 83 * At this point: 84 * x28 = Our physical load address 85 */ 86 87 /* Create the page tables */ 88 bl create_pagetables 89 90 /* 91 * At this point: 92 * x27 = TTBR0 table 93 * x26 = Kernel L1 table 94 * x24 = TTBR1 table 95 */ 96 97 /* Enable the mmu */ 98 bl start_mmu 99 100 /* Load the new ttbr0 pagetable */ 101 adrp x27, pagetable_l0_ttbr0 102 add x27, x27, :lo12:pagetable_l0_ttbr0 103 104 /* Jump to the virtual address space */ 105 ldr x15, .Lvirtdone 106 br x15 107 108virtdone: 109 BTI_J 110 111 /* Set up the stack */ 112 adrp x25, initstack_end 113 add x25, x25, :lo12:initstack_end 114 sub sp, x25, #PCB_SIZE 115 116 /* Zero the BSS */ 117 ldr x15, .Lbss 118 ldr x14, .Lend 1191: 120 str xzr, [x15], #8 121 cmp x15, x14 122 b.lo 1b 123 124#if defined(PERTHREAD_SSP) 125 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 126 adrp x15, boot_canary 127 add x15, x15, :lo12:boot_canary 128 msr sp_el0, x15 129#endif 130 131 /* Backup the module pointer */ 132 mov x1, x0 133 134 sub sp, sp, #BOOTPARAMS_SIZE 135 mov x0, sp 136 137 /* Negate the delta so it is VA -> PA */ 138 139 str x1, [x0, #BP_MODULEP] 140 adrp x25, initstack 141 add x25, x25, :lo12:initstack 142 str x25, [x0, #BP_KERN_STACK] 143 str x27, [x0, #BP_KERN_TTBR0] 144 str x23, [x0, #BP_BOOT_EL] 145 str x4, [x0, #BP_HCR_EL2] 146 147#ifdef KASAN 148 /* Save bootparams */ 149 mov x19, x0 150 151 /* Bootstrap an early shadow map for the boot stack. */ 152 bl pmap_san_bootstrap 153 154 /* Restore bootparams */ 155 mov x0, x19 156#endif 157 158 /* trace back starts here */ 159 mov fp, #0 160 /* Branch to C code */ 161 bl initarm 162 /* We are done with the boot params */ 163 add sp, sp, #BOOTPARAMS_SIZE 164 165 /* 166 * Enable pointer authentication in the kernel. We set the keys for 167 * thread0 in initarm so have to wait until it returns to enable it. 168 * If we were to enable it in initarm then any authentication when 169 * returning would fail as it was called with pointer authentication 170 * disabled. 171 */ 172 bl ptrauth_start 173 174 bl mi_startup 175 176 /* We should not get here */ 177 brk 0 178 179 .align 3 180.Lvirtdone: 181 .quad virtdone 182.Lbss: 183 .quad __bss_start 184.Lend: 185 .quad __bss_end 186END(_start) 187 188#ifdef SMP 189/* 190 * mpentry(unsigned long) 191 * 192 * Called by a core when it is being brought online. 193 * The data in x0 is passed straight to init_secondary. 194 */ 195ENTRY(mpentry) 196 /* Disable interrupts */ 197 msr daifset, #DAIF_INTR 198 199 /* Drop to EL1 */ 200 bl drop_to_el1 201 202 /* Set the context id */ 203 msr contextidr_el1, xzr 204 205 /* Load the kernel page table */ 206 adrp x24, pagetable_l0_ttbr1 207 add x24, x24, :lo12:pagetable_l0_ttbr1 208 /* Load the identity page table */ 209 adrp x27, pagetable_l0_ttbr0_boostrap 210 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 211 212 /* Enable the mmu */ 213 bl start_mmu 214 215 /* Load the new ttbr0 pagetable */ 216 adrp x27, pagetable_l0_ttbr0 217 add x27, x27, :lo12:pagetable_l0_ttbr0 218 219 /* Jump to the virtual address space */ 220 ldr x15, =mp_virtdone 221 br x15 222 223mp_virtdone: 224 BTI_J 225 226 /* Start using the AP boot stack */ 227 adrp x4, bootstack 228 ldr x4, [x4, :lo12:bootstack] 229 mov sp, x4 230 231#if defined(PERTHREAD_SSP) 232 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 233 adrp x15, boot_canary 234 add x15, x15, :lo12:boot_canary 235 msr sp_el0, x15 236#endif 237 238 /* Load the kernel ttbr0 pagetable */ 239 msr ttbr0_el1, x27 240 isb 241 242 /* Invalidate the TLB */ 243 tlbi vmalle1 244 dsb sy 245 isb 246 247 /* 248 * Initialize the per-CPU pointer before calling into C code, for the 249 * benefit of kernel sanitizers. 250 */ 251 adrp x18, bootpcpu 252 ldr x18, [x18, :lo12:bootpcpu] 253 msr tpidr_el1, x18 254 255 b init_secondary 256END(mpentry) 257#endif 258 259/* 260 * If we are started in EL2, configure the required hypervisor 261 * registers and drop to EL1. 262 */ 263LENTRY(drop_to_el1) 264 mrs x23, CurrentEL 265 lsr x23, x23, #2 266 cmp x23, #0x2 267 b.eq 1f 268 ret 2691: 270 /* 271 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 272 * which may break address translation. 273 */ 274 dsb sy 275 mrs x2, sctlr_el2 276 bic x2, x2, SCTLR_M 277 msr sctlr_el2, x2 278 isb 279 280 /* Configure the Hypervisor */ 281 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 282 msr hcr_el2, x2 283 284 /* Stash value of HCR_EL2 for later */ 285 isb 286 mrs x4, hcr_el2 287 288 /* Load the Virtualization Process ID Register */ 289 mrs x2, midr_el1 290 msr vpidr_el2, x2 291 292 /* Load the Virtualization Multiprocess ID Register */ 293 mrs x2, mpidr_el1 294 msr vmpidr_el2, x2 295 296 /* Set the bits that need to be 1 in sctlr_el1 */ 297 ldr x2, .Lsctlr_res1 298 msr sctlr_el1, x2 299 300 /* 301 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 302 * don't trap to EL2 for SIMD register usage to have at least a 303 * minimally usable system. 304 */ 305 tst x4, #HCR_E2H 306 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 307 mov x5, #CPTR_FPEN /* HCR_E2H == 1 */ 308 csel x2, x3, x5, eq 309 msr cptr_el2, x2 310 311 /* Don't trap to EL2 for CP15 traps */ 312 msr hstr_el2, xzr 313 314 /* Enable access to the physical timers at EL1 */ 315 mrs x2, cnthctl_el2 316 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 317 msr cnthctl_el2, x2 318 319 /* Set the counter offset to a known value */ 320 msr cntvoff_el2, xzr 321 322 /* Hypervisor trap functions */ 323 adrp x2, hyp_stub_vectors 324 add x2, x2, :lo12:hyp_stub_vectors 325 msr vbar_el2, x2 326 327 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 328 msr vttbr_el2, xzr 329 330 mov x2, #(PSR_DAIF | PSR_M_EL1h) 331 msr spsr_el2, x2 332 333 /* Configure GICv3 CPU interface */ 334 mrs x2, id_aa64pfr0_el1 335 /* Extract GIC bits from the register */ 336 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 337 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 338 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 339 b.ne 2f 340 341 mrs x2, icc_sre_el2 342 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 343 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 344 msr icc_sre_el2, x2 3452: 346 347 /* Set the address to return to our return address */ 348 msr elr_el2, x30 349 isb 350 351 eret 352 353 .align 3 354.Lsctlr_res1: 355 .quad SCTLR_RES1 356LEND(drop_to_el1) 357 358/* 359 * Get the physical address the kernel was loaded at. 360 */ 361LENTRY(get_load_phys_addr) 362 /* Load the offset of get_load_phys_addr from KERNBASE */ 363 ldr x28, =(get_load_phys_addr - KERNBASE) 364 /* Load the physical address of get_load_phys_addr */ 365 adr x29, get_load_phys_addr 366 /* Find the physical address of KERNBASE, i.e. our load address */ 367 sub x28, x29, x28 368 ret 369LEND(get_load_phys_addr) 370 371/* 372 * This builds the page tables containing the identity map, and the kernel 373 * virtual map. 374 * 375 * It relys on: 376 * We were loaded to an address that is on a 2MiB boundary 377 * All the memory must not cross a 1GiB boundaty 378 * x28 contains the physical address we were loaded from 379 * 380 * TODO: This is out of date. 381 * There are at least 5 pages before that address for the page tables 382 * The pages used are: 383 * - The Kernel L2 table 384 * - The Kernel L1 table 385 * - The Kernel L0 table (TTBR1) 386 * - The identity (PA = VA) L1 table 387 * - The identity (PA = VA) L0 table (TTBR0) 388 */ 389LENTRY(create_pagetables) 390 /* Save the Link register */ 391 mov x5, x30 392 393 /* Clean the page table */ 394 adrp x6, pagetable 395 add x6, x6, :lo12:pagetable 396 mov x26, x6 397 adrp x27, pagetable_end 398 add x27, x27, :lo12:pagetable_end 3991: 400 stp xzr, xzr, [x6], #16 401 stp xzr, xzr, [x6], #16 402 stp xzr, xzr, [x6], #16 403 stp xzr, xzr, [x6], #16 404 cmp x6, x27 405 b.lo 1b 406 407 /* 408 * Build the TTBR1 maps. 409 */ 410 411 /* Find the size of the kernel */ 412 mov x6, #(KERNBASE) 413 414#if defined(LINUX_BOOT_ABI) 415 /* X19 is used as 'map FDT data' flag */ 416 mov x19, xzr 417 418 /* No modules or FDT pointer ? */ 419 cbz x0, booti_no_fdt 420 421 /* 422 * Test if x0 points to modules descriptor(virtual address) or 423 * to FDT (physical address) 424 */ 425 cmp x0, x6 /* x6 is #(KERNBASE) */ 426 b.lo booti_fdt 427#endif 428 429 /* Booted with modules pointer */ 430 /* Find modulep - begin */ 431 sub x8, x0, x6 432 /* 433 * Add space for the module data. When PAGE_SIZE is 4k this will 434 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 435 * larger it will be at least as large as we use smaller level 3 436 * pages. 437 */ 438 ldr x7, =((6 * 1024 * 1024) - 1) 439 add x8, x8, x7 440 b common 441 442#if defined(LINUX_BOOT_ABI) 443booti_fdt: 444 /* Booted by U-Boot booti with FDT data */ 445 /* Set 'map FDT data' flag */ 446 mov x19, #1 447 448booti_no_fdt: 449 /* Booted by U-Boot booti without FTD data */ 450 /* Find the end - begin */ 451 ldr x7, .Lend 452 sub x8, x7, x6 453 454 /* 455 * Add one 2MiB page for copy of FDT data (maximum FDT size), 456 * one for metadata and round up 457 */ 458 ldr x7, =(3 * L2_SIZE - 1) 459 add x8, x8, x7 460#endif 461 462common: 463#if PAGE_SIZE != PAGE_SIZE_4K 464 /* 465 * Create L3 pages. The kernel will be loaded at a 2M aligned 466 * address, however L2 blocks are too large when the page size is 467 * not 4k to map the kernel with such an aligned address. However, 468 * when the page size is larger than 4k, L2 blocks are too large to 469 * map the kernel with such an alignment. 470 */ 471 472 /* Get the number of l3 pages to allocate, rounded down */ 473 lsr x10, x8, #(L3_SHIFT) 474 475 /* Create the kernel space L2 table */ 476 mov x6, x26 477 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 478 mov x8, #(KERNBASE) 479 mov x9, x28 480 bl build_l3_page_pagetable 481 482 /* Move to the l2 table */ 483 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 484 add x26, x26, x9 485 486 /* Link the l2 -> l3 table */ 487 mov x9, x6 488 mov x6, x26 489 bl link_l2_pagetable 490#else 491 /* Get the number of l2 pages to allocate, rounded down */ 492 lsr x10, x8, #(L2_SHIFT) 493 494 /* Create the kernel space L2 table */ 495 mov x6, x26 496 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 497 mov x8, #(KERNBASE) 498 mov x9, x28 499 bl build_l2_block_pagetable 500#endif 501 502 /* Move to the l1 table */ 503 add x26, x26, #PAGE_SIZE 504 505 /* Link the l1 -> l2 table */ 506 mov x9, x6 507 mov x6, x26 508 bl link_l1_pagetable 509 510 /* Move to the l0 table */ 511 add x24, x26, #PAGE_SIZE 512 513 /* Link the l0 -> l1 table */ 514 mov x9, x6 515 mov x6, x24 516 mov x10, #1 517 bl link_l0_pagetable 518 519 /* 520 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 521 * They are only needed early on, so the VA = PA map is uncached. 522 */ 523 add x27, x24, #PAGE_SIZE 524 525 mov x6, x27 /* The initial page table */ 526 527 /* Create the VA = PA map */ 528 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 529 adrp x16, _start 530 and x16, x16, #(~L2_OFFSET) 531 mov x9, x16 /* PA start */ 532 mov x8, x16 /* VA start (== PA start) */ 533 mov x10, #1 534 bl build_l2_block_pagetable 535 536#if defined(SOCDEV_PA) 537 /* Create a table for the UART */ 538 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 539 ldr x9, =(L2_SIZE) 540 add x16, x16, x9 /* VA start */ 541 mov x8, x16 542 543 /* Store the socdev virtual address */ 544 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 545 adrp x9, socdev_va 546 str x17, [x9, :lo12:socdev_va] 547 548 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 549 mov x10, #1 550 bl build_l2_block_pagetable 551#endif 552 553#if defined(LINUX_BOOT_ABI) 554 /* Map FDT data ? */ 555 cbz x19, 1f 556 557 /* Create the mapping for FDT data (2 MiB max) */ 558 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 559 ldr x9, =(L2_SIZE) 560 add x16, x16, x9 /* VA start */ 561 mov x8, x16 562 mov x9, x0 /* PA start */ 563 /* Update the module pointer to point at the allocated memory */ 564 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 565 add x0, x0, x8 /* Add the aligned virtual address */ 566 567 mov x10, #1 568 bl build_l2_block_pagetable 569 5701: 571#endif 572 573 /* Move to the l1 table */ 574 add x27, x27, #PAGE_SIZE 575 576 /* Link the l1 -> l2 table */ 577 mov x9, x6 578 mov x6, x27 579 bl link_l1_pagetable 580 581 /* Move to the l0 table */ 582 add x27, x27, #PAGE_SIZE 583 584 /* Link the l0 -> l1 table */ 585 mov x9, x6 586 mov x6, x27 587 mov x10, #1 588 bl link_l0_pagetable 589 590 /* Restore the Link register */ 591 mov x30, x5 592 ret 593LEND(create_pagetables) 594 595/* 596 * Builds an L0 -> L1 table descriptor 597 * 598 * x6 = L0 table 599 * x8 = Virtual Address 600 * x9 = L1 PA (trashed) 601 * x10 = Entry count (trashed) 602 * x11, x12 and x13 are trashed 603 */ 604LENTRY(link_l0_pagetable) 605 /* 606 * Link an L0 -> L1 table entry. 607 */ 608 /* Find the table index */ 609 lsr x11, x8, #L0_SHIFT 610 and x11, x11, #L0_ADDR_MASK 611 612 /* Build the L0 block entry */ 613 mov x12, #L0_TABLE 614 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 615 616 /* Only use the output address bits */ 617 lsr x9, x9, #PAGE_SHIFT 6181: orr x13, x12, x9, lsl #PAGE_SHIFT 619 620 /* Store the entry */ 621 str x13, [x6, x11, lsl #3] 622 623 sub x10, x10, #1 624 add x11, x11, #1 625 add x9, x9, #1 626 cbnz x10, 1b 627 628 ret 629LEND(link_l0_pagetable) 630 631/* 632 * Builds an L1 -> L2 table descriptor 633 * 634 * x6 = L1 table 635 * x8 = Virtual Address 636 * x9 = L2 PA (trashed) 637 * x11, x12 and x13 are trashed 638 */ 639LENTRY(link_l1_pagetable) 640 /* 641 * Link an L1 -> L2 table entry. 642 */ 643 /* Find the table index */ 644 lsr x11, x8, #L1_SHIFT 645 and x11, x11, #Ln_ADDR_MASK 646 647 /* Build the L1 block entry */ 648 mov x12, #L1_TABLE 649 650 /* Only use the output address bits */ 651 lsr x9, x9, #PAGE_SHIFT 652 orr x13, x12, x9, lsl #PAGE_SHIFT 653 654 /* Store the entry */ 655 str x13, [x6, x11, lsl #3] 656 657 ret 658LEND(link_l1_pagetable) 659 660/* 661 * Builds count 2 MiB page table entry 662 * x6 = L2 table 663 * x7 = Block attributes 664 * x8 = VA start 665 * x9 = PA start (trashed) 666 * x10 = Entry count (trashed) 667 * x11, x12 and x13 are trashed 668 */ 669LENTRY(build_l2_block_pagetable) 670 /* 671 * Build the L2 table entry. 672 */ 673 /* Find the table index */ 674 lsr x11, x8, #L2_SHIFT 675 and x11, x11, #Ln_ADDR_MASK 676 677 /* Build the L2 block entry */ 678 orr x12, x7, #L2_BLOCK 679 orr x12, x12, #(ATTR_DEFAULT) 680 orr x12, x12, #(ATTR_S1_UXN) 681 682 /* Only use the output address bits */ 683 lsr x9, x9, #L2_SHIFT 684 685 /* Set the physical address for this virtual address */ 6861: orr x13, x12, x9, lsl #L2_SHIFT 687 688 /* Store the entry */ 689 str x13, [x6, x11, lsl #3] 690 691 sub x10, x10, #1 692 add x11, x11, #1 693 add x9, x9, #1 694 cbnz x10, 1b 695 696 ret 697LEND(build_l2_block_pagetable) 698 699#if PAGE_SIZE != PAGE_SIZE_4K 700/* 701 * Builds an L2 -> L3 table descriptor 702 * 703 * x6 = L2 table 704 * x8 = Virtual Address 705 * x9 = L3 PA (trashed) 706 * x11, x12 and x13 are trashed 707 */ 708LENTRY(link_l2_pagetable) 709 /* 710 * Link an L2 -> L3 table entry. 711 */ 712 /* Find the table index */ 713 lsr x11, x8, #L2_SHIFT 714 and x11, x11, #Ln_ADDR_MASK 715 716 /* Build the L1 block entry */ 717 mov x12, #L2_TABLE 718 719 /* Only use the output address bits */ 720 lsr x9, x9, #PAGE_SHIFT 721 orr x13, x12, x9, lsl #PAGE_SHIFT 722 723 /* Store the entry */ 724 str x13, [x6, x11, lsl #3] 725 726 ret 727LEND(link_l2_pagetable) 728 729/* 730 * Builds count level 3 page table entries 731 * x6 = L3 table 732 * x7 = Block attributes 733 * x8 = VA start 734 * x9 = PA start (trashed) 735 * x10 = Entry count (trashed) 736 * x11, x12 and x13 are trashed 737 */ 738LENTRY(build_l3_page_pagetable) 739 /* 740 * Build the L3 table entry. 741 */ 742 /* Find the table index */ 743 lsr x11, x8, #L3_SHIFT 744 and x11, x11, #Ln_ADDR_MASK 745 746 /* Build the L3 page entry */ 747 orr x12, x7, #L3_PAGE 748 orr x12, x12, #(ATTR_DEFAULT) 749 orr x12, x12, #(ATTR_S1_UXN) 750 751 /* Only use the output address bits */ 752 lsr x9, x9, #L3_SHIFT 753 754 /* Set the physical address for this virtual address */ 7551: orr x13, x12, x9, lsl #L3_SHIFT 756 757 /* Store the entry */ 758 str x13, [x6, x11, lsl #3] 759 760 sub x10, x10, #1 761 add x11, x11, #1 762 add x9, x9, #1 763 cbnz x10, 1b 764 765 ret 766LEND(build_l3_page_pagetable) 767#endif 768 769LENTRY(start_mmu) 770 dsb sy 771 772 /* Load the exception vectors */ 773 ldr x2, =exception_vectors 774 msr vbar_el1, x2 775 776 /* Load ttbr0 and ttbr1 */ 777 msr ttbr0_el1, x27 778 msr ttbr1_el1, x24 779 isb 780 781 /* Clear the Monitor Debug System control register */ 782 msr mdscr_el1, xzr 783 784 /* Invalidate the TLB */ 785 tlbi vmalle1is 786 dsb ish 787 isb 788 789 ldr x2, mair 790 msr mair_el1, x2 791 792 /* 793 * Setup TCR according to the PARange and ASIDBits fields 794 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 795 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 796 * to 1 only if the ASIDBits field equals 0b0010. 797 */ 798 ldr x2, tcr 799 mrs x3, id_aa64mmfr0_el1 800 801 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 802 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 803 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 804 805 /* Check if the HW supports 16 bit ASIDS */ 806 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 807 /* If so x3 == 1, else x3 == 0 */ 808 cset x3, eq 809 /* Set TCR.AS with x3 */ 810 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 811 812 /* 813 * Check if the HW supports access flag and dirty state updates, 814 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 815 */ 816 mrs x3, id_aa64mmfr1_el1 817 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 818 cmp x3, #1 819 b.ne 1f 820 orr x2, x2, #(TCR_HA) 821 b 2f 8221: 823 cmp x3, #2 824 b.ne 2f 825 orr x2, x2, #(TCR_HA | TCR_HD) 8262: 827 msr tcr_el1, x2 828 829 /* 830 * Setup SCTLR. 831 */ 832 ldr x2, sctlr_set 833 ldr x3, sctlr_clear 834 mrs x1, sctlr_el1 835 bic x1, x1, x3 /* Clear the required bits */ 836 orr x1, x1, x2 /* Set the required bits */ 837 msr sctlr_el1, x1 838 isb 839 840 ret 841 842 .align 3 843mair: 844 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 845 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 846 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 847 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 848 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 849tcr: 850#if PAGE_SIZE == PAGE_SIZE_4K 851#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 852#elif PAGE_SIZE == PAGE_SIZE_16K 853#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 854#else 855#error Unsupported page size 856#endif 857 858 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 859 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 860sctlr_set: 861 /* Bits to set */ 862 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 863 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 864 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 865 SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 866sctlr_clear: 867 /* Bits to clear */ 868 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 869 SCTLR_ITD | SCTLR_A) 870LEND(start_mmu) 871 872ENTRY(abort) 873 b abort 874END(abort) 875 876.bss 877 .align PAGE_SHIFT 878initstack: 879 .space (PAGE_SIZE * KSTACK_PAGES) 880initstack_end: 881 882 .section .init_pagetable, "aw", %nobits 883 .align PAGE_SHIFT 884 /* 885 * 6 initial tables (in the following order): 886 * L2 for kernel (High addresses) 887 * L1 for kernel 888 * L0 for kernel 889 * L1 bootstrap for user (Low addresses) 890 * L0 bootstrap for user 891 * L0 for user 892 */ 893 .globl pagetable_l0_ttbr1 894pagetable: 895#if PAGE_SIZE != PAGE_SIZE_4K 896 .space (PAGE_SIZE * L3_PAGE_COUNT) 897pagetable_l2_ttbr1: 898#endif 899 .space PAGE_SIZE 900pagetable_l1_ttbr1: 901 .space PAGE_SIZE 902pagetable_l0_ttbr1: 903 .space PAGE_SIZE 904pagetable_l2_ttbr0_bootstrap: 905 .space PAGE_SIZE 906pagetable_l1_ttbr0_bootstrap: 907 .space PAGE_SIZE 908pagetable_l0_ttbr0_boostrap: 909 .space PAGE_SIZE 910pagetable_l0_ttbr0: 911 .space PAGE_SIZE 912pagetable_end: 913 914el2_pagetable: 915 .space PAGE_SIZE 916 917 .section .rodata, "a", %progbits 918 .globl aarch32_sigcode 919 .align 2 920aarch32_sigcode: 921 .word 0xe1a0000d // mov r0, sp 922 .word 0xe2800040 // add r0, r0, #SIGF_UC 923 .word 0xe59f700c // ldr r7, [pc, #12] 924 .word 0xef000000 // swi #0 925 .word 0xe59f7008 // ldr r7, [pc, #8] 926 .word 0xef000000 // swi #0 927 .word 0xeafffffa // b . - 16 928 .word SYS_sigreturn 929 .word SYS_exit 930 .align 3 931 .size aarch32_sigcode, . - aarch32_sigcode 932aarch32_esigcode: 933 .data 934 .global sz_aarch32_sigcode 935sz_aarch32_sigcode: 936 .quad aarch32_esigcode - aarch32_sigcode 937