1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include "assym.inc" 28#include "opt_kstack_pages.h" 29#include <sys/syscall.h> 30#include <machine/asm.h> 31#include <machine/armreg.h> 32#include <machine/hypervisor.h> 33#include <machine/param.h> 34#include <machine/pte.h> 35#include <machine/vm.h> 36#include <machine/vmparam.h> 37 38#define VIRT_BITS 48 39 40#if PAGE_SIZE == PAGE_SIZE_16K 41/* 42 * The number of level 3 tables to create. 32 will allow for 1G of address 43 * space, the same as a single level 2 page with 4k pages. 44 */ 45#define L3_PAGE_COUNT 32 46#endif 47 48 .globl kernbase 49 .set kernbase, KERNBASE 50 51/* 52 * We assume: 53 * MMU on with an identity map, or off 54 * D-Cache: off 55 * I-Cache: on or off 56 * We are loaded at a 2MiB aligned address 57 */ 58 59ENTRY(_start) 60 /* Drop to EL1 */ 61 bl drop_to_el1 62 63 /* 64 * Disable the MMU. We may have entered the kernel with it on and 65 * will need to update the tables later. If this has been set up 66 * with anything other than a VA == PA map then this will fail, 67 * but in this case the code to find where we are running from 68 * would have also failed. 69 */ 70 dsb sy 71 mrs x2, sctlr_el1 72 bic x2, x2, SCTLR_M 73 msr sctlr_el1, x2 74 isb 75 76 /* Set the context id */ 77 msr contextidr_el1, xzr 78 79 /* Get the virt -> phys offset */ 80 bl get_virt_delta 81 82 /* 83 * At this point: 84 * x29 = PA - VA 85 * x28 = Our physical load address 86 */ 87 88 /* Create the page tables */ 89 bl create_pagetables 90 91 /* 92 * At this point: 93 * x27 = TTBR0 table 94 * x26 = Kernel L1 table 95 * x24 = TTBR1 table 96 */ 97 98 /* Enable the mmu */ 99 bl start_mmu 100 101 /* Load the new ttbr0 pagetable */ 102 adrp x27, pagetable_l0_ttbr0 103 add x27, x27, :lo12:pagetable_l0_ttbr0 104 105 /* Jump to the virtual address space */ 106 ldr x15, .Lvirtdone 107 br x15 108 109virtdone: 110 /* Set up the stack */ 111 adrp x25, initstack_end 112 add x25, x25, :lo12:initstack_end 113 mov sp, x25 114 sub sp, sp, #PCB_SIZE 115 116 /* Zero the BSS */ 117 ldr x15, .Lbss 118 ldr x14, .Lend 1191: 120 str xzr, [x15], #8 121 cmp x15, x14 122 b.lo 1b 123 124#if defined(PERTHREAD_SSP) 125 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 126 adrp x15, boot_canary 127 add x15, x15, :lo12:boot_canary 128 msr sp_el0, x15 129#endif 130 131 /* Backup the module pointer */ 132 mov x1, x0 133 134 sub sp, sp, #BOOTPARAMS_SIZE 135 mov x0, sp 136 137 /* Degate the delda so it is VA -> PA */ 138 neg x29, x29 139 140 str x1, [x0, #BP_MODULEP] 141 str x29, [x0, #BP_KERN_DELTA] 142 adrp x25, initstack 143 add x25, x25, :lo12:initstack 144 str x25, [x0, #BP_KERN_STACK] 145 str x27, [x0, #BP_KERN_TTBR0] 146 str x23, [x0, #BP_BOOT_EL] 147 str x4, [x0, #BP_HCR_EL2] 148 149#ifdef KASAN 150 /* Save bootparams */ 151 mov x19, x0 152 153 /* Bootstrap an early shadow map for the boot stack. */ 154 bl pmap_san_bootstrap 155 156 /* Restore bootparams */ 157 mov x0, x19 158#endif 159 160 /* trace back starts here */ 161 mov fp, #0 162 /* Branch to C code */ 163 bl initarm 164 /* We are done with the boot params */ 165 add sp, sp, #BOOTPARAMS_SIZE 166 167 /* 168 * Enable pointer authentication in the kernel. We set the keys for 169 * thread0 in initarm so have to wait until it returns to enable it. 170 * If we were to enable it in initarm then any authentication when 171 * returning would fail as it was called with pointer authentication 172 * disabled. 173 */ 174 bl ptrauth_start 175 176 bl mi_startup 177 178 /* We should not get here */ 179 brk 0 180 181 .align 3 182.Lvirtdone: 183 .quad virtdone 184.Lbss: 185 .quad __bss_start 186.Lend: 187 .quad __bss_end 188END(_start) 189 190#ifdef SMP 191/* 192 * mpentry(unsigned long) 193 * 194 * Called by a core when it is being brought online. 195 * The data in x0 is passed straight to init_secondary. 196 */ 197ENTRY(mpentry) 198 /* Disable interrupts */ 199 msr daifset, #DAIF_INTR 200 201 /* Drop to EL1 */ 202 bl drop_to_el1 203 204 /* Set the context id */ 205 msr contextidr_el1, xzr 206 207 /* Load the kernel page table */ 208 adrp x24, pagetable_l0_ttbr1 209 add x24, x24, :lo12:pagetable_l0_ttbr1 210 /* Load the identity page table */ 211 adrp x27, pagetable_l0_ttbr0_boostrap 212 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 213 214 /* Enable the mmu */ 215 bl start_mmu 216 217 /* Load the new ttbr0 pagetable */ 218 adrp x27, pagetable_l0_ttbr0 219 add x27, x27, :lo12:pagetable_l0_ttbr0 220 221 /* Jump to the virtual address space */ 222 ldr x15, =mp_virtdone 223 br x15 224 225mp_virtdone: 226 /* Start using the AP boot stack */ 227 ldr x4, =bootstack 228 ldr x4, [x4] 229 mov sp, x4 230 231#if defined(PERTHREAD_SSP) 232 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 233 adrp x15, boot_canary 234 add x15, x15, :lo12:boot_canary 235 msr sp_el0, x15 236#endif 237 238 /* Load the kernel ttbr0 pagetable */ 239 msr ttbr0_el1, x27 240 isb 241 242 /* Invalidate the TLB */ 243 tlbi vmalle1 244 dsb sy 245 isb 246 247 b init_secondary 248END(mpentry) 249#endif 250 251/* 252 * If we are started in EL2, configure the required hypervisor 253 * registers and drop to EL1. 254 */ 255LENTRY(drop_to_el1) 256 mrs x23, CurrentEL 257 lsr x23, x23, #2 258 cmp x23, #0x2 259 b.eq 1f 260 ret 2611: 262 /* 263 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 264 * which may break address translation. 265 */ 266 dsb sy 267 mrs x2, sctlr_el2 268 bic x2, x2, SCTLR_M 269 msr sctlr_el2, x2 270 isb 271 272 /* Configure the Hypervisor */ 273 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 274 msr hcr_el2, x2 275 276 /* Stash value of HCR_EL2 for later */ 277 isb 278 mrs x4, hcr_el2 279 280 /* Load the Virtualization Process ID Register */ 281 mrs x2, midr_el1 282 msr vpidr_el2, x2 283 284 /* Load the Virtualization Multiprocess ID Register */ 285 mrs x2, mpidr_el1 286 msr vmpidr_el2, x2 287 288 /* Set the bits that need to be 1 in sctlr_el1 */ 289 ldr x2, .Lsctlr_res1 290 msr sctlr_el1, x2 291 292 /* 293 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 294 * don't trap to EL2 for SIMD register usage to have at least a 295 * minimally usable system. 296 */ 297 tst x4, #HCR_E2H 298 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 299 mov x5, #CPTR_FPEN /* HCR_E2H == 1 */ 300 csel x2, x3, x5, eq 301 msr cptr_el2, x2 302 303 /* Don't trap to EL2 for CP15 traps */ 304 msr hstr_el2, xzr 305 306 /* Enable access to the physical timers at EL1 */ 307 mrs x2, cnthctl_el2 308 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 309 msr cnthctl_el2, x2 310 311 /* Set the counter offset to a known value */ 312 msr cntvoff_el2, xzr 313 314 /* Hypervisor trap functions */ 315 adrp x2, hyp_stub_vectors 316 add x2, x2, :lo12:hyp_stub_vectors 317 msr vbar_el2, x2 318 319 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 320 msr vttbr_el2, xzr 321 322 mov x2, #(PSR_DAIF | PSR_M_EL1h) 323 msr spsr_el2, x2 324 325 /* Configure GICv3 CPU interface */ 326 mrs x2, id_aa64pfr0_el1 327 /* Extract GIC bits from the register */ 328 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 329 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 330 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 331 b.ne 2f 332 333 mrs x2, icc_sre_el2 334 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 335 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 336 msr icc_sre_el2, x2 3372: 338 339 /* Set the address to return to our return address */ 340 msr elr_el2, x30 341 isb 342 343 eret 344 345 .align 3 346.Lsctlr_res1: 347 .quad SCTLR_RES1 348LEND(drop_to_el1) 349 350/* 351 * Get the delta between the physical address we were loaded to and the 352 * virtual address we expect to run from. This is used when building the 353 * initial page table. 354 */ 355LENTRY(get_virt_delta) 356 /* Load the physical address of virt_map */ 357 adrp x29, virt_map 358 add x29, x29, :lo12:virt_map 359 /* Load the virtual address of virt_map stored in virt_map */ 360 ldr x28, [x29] 361 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 362 sub x29, x29, x28 363 /* Find the load address for the kernel */ 364 mov x28, #(KERNBASE) 365 add x28, x28, x29 366 ret 367 368 .align 3 369virt_map: 370 .quad virt_map 371LEND(get_virt_delta) 372 373/* 374 * This builds the page tables containing the identity map, and the kernel 375 * virtual map. 376 * 377 * It relys on: 378 * We were loaded to an address that is on a 2MiB boundary 379 * All the memory must not cross a 1GiB boundaty 380 * x28 contains the physical address we were loaded from 381 * 382 * TODO: This is out of date. 383 * There are at least 5 pages before that address for the page tables 384 * The pages used are: 385 * - The Kernel L2 table 386 * - The Kernel L1 table 387 * - The Kernel L0 table (TTBR1) 388 * - The identity (PA = VA) L1 table 389 * - The identity (PA = VA) L0 table (TTBR0) 390 */ 391LENTRY(create_pagetables) 392 /* Save the Link register */ 393 mov x5, x30 394 395 /* Clean the page table */ 396 adrp x6, pagetable 397 add x6, x6, :lo12:pagetable 398 mov x26, x6 399 adrp x27, pagetable_end 400 add x27, x27, :lo12:pagetable_end 4011: 402 stp xzr, xzr, [x6], #16 403 stp xzr, xzr, [x6], #16 404 stp xzr, xzr, [x6], #16 405 stp xzr, xzr, [x6], #16 406 cmp x6, x27 407 b.lo 1b 408 409 /* 410 * Build the TTBR1 maps. 411 */ 412 413 /* Find the size of the kernel */ 414 mov x6, #(KERNBASE) 415 416#if defined(LINUX_BOOT_ABI) 417 /* X19 is used as 'map FDT data' flag */ 418 mov x19, xzr 419 420 /* No modules or FDT pointer ? */ 421 cbz x0, booti_no_fdt 422 423 /* 424 * Test if x0 points to modules descriptor(virtual address) or 425 * to FDT (physical address) 426 */ 427 cmp x0, x6 /* x6 is #(KERNBASE) */ 428 b.lo booti_fdt 429#endif 430 431 /* Booted with modules pointer */ 432 /* Find modulep - begin */ 433 sub x8, x0, x6 434 /* 435 * Add space for the module data. When PAGE_SIZE is 4k this will 436 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 437 * larger it will be at least as large as we use smaller level 3 438 * pages. 439 */ 440 ldr x7, =((6 * 1024 * 1024) - 1) 441 add x8, x8, x7 442 b common 443 444#if defined(LINUX_BOOT_ABI) 445booti_fdt: 446 /* Booted by U-Boot booti with FDT data */ 447 /* Set 'map FDT data' flag */ 448 mov x19, #1 449 450booti_no_fdt: 451 /* Booted by U-Boot booti without FTD data */ 452 /* Find the end - begin */ 453 ldr x7, .Lend 454 sub x8, x7, x6 455 456 /* 457 * Add one 2MiB page for copy of FDT data (maximum FDT size), 458 * one for metadata and round up 459 */ 460 ldr x7, =(3 * L2_SIZE - 1) 461 add x8, x8, x7 462#endif 463 464common: 465#if PAGE_SIZE != PAGE_SIZE_4K 466 /* 467 * Create L3 pages. The kernel will be loaded at a 2M aligned 468 * address, however L2 blocks are too large when the page size is 469 * not 4k to map the kernel with such an aligned address. However, 470 * when the page size is larger than 4k, L2 blocks are too large to 471 * map the kernel with such an alignment. 472 */ 473 474 /* Get the number of l3 pages to allocate, rounded down */ 475 lsr x10, x8, #(L3_SHIFT) 476 477 /* Create the kernel space L2 table */ 478 mov x6, x26 479 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 480 mov x8, #(KERNBASE) 481 mov x9, x28 482 bl build_l3_page_pagetable 483 484 /* Move to the l2 table */ 485 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 486 add x26, x26, x9 487 488 /* Link the l2 -> l3 table */ 489 mov x9, x6 490 mov x6, x26 491 bl link_l2_pagetable 492#else 493 /* Get the number of l2 pages to allocate, rounded down */ 494 lsr x10, x8, #(L2_SHIFT) 495 496 /* Create the kernel space L2 table */ 497 mov x6, x26 498 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 499 mov x8, #(KERNBASE) 500 mov x9, x28 501 bl build_l2_block_pagetable 502#endif 503 504 /* Move to the l1 table */ 505 add x26, x26, #PAGE_SIZE 506 507 /* Link the l1 -> l2 table */ 508 mov x9, x6 509 mov x6, x26 510 bl link_l1_pagetable 511 512 /* Move to the l0 table */ 513 add x24, x26, #PAGE_SIZE 514 515 /* Link the l0 -> l1 table */ 516 mov x9, x6 517 mov x6, x24 518 mov x10, #1 519 bl link_l0_pagetable 520 521 /* 522 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 523 * They are only needed early on, so the VA = PA map is uncached. 524 */ 525 add x27, x24, #PAGE_SIZE 526 527 mov x6, x27 /* The initial page table */ 528 529 /* Create the VA = PA map */ 530 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 531 adrp x16, _start 532 and x16, x16, #(~L2_OFFSET) 533 mov x9, x16 /* PA start */ 534 mov x8, x16 /* VA start (== PA start) */ 535 mov x10, #1 536 bl build_l2_block_pagetable 537 538#if defined(SOCDEV_PA) 539 /* Create a table for the UART */ 540 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 541 ldr x9, =(L2_SIZE) 542 add x16, x16, x9 /* VA start */ 543 mov x8, x16 544 545 /* Store the socdev virtual address */ 546 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 547 adrp x9, socdev_va 548 str x17, [x9, :lo12:socdev_va] 549 550 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 551 mov x10, #1 552 bl build_l2_block_pagetable 553#endif 554 555#if defined(LINUX_BOOT_ABI) 556 /* Map FDT data ? */ 557 cbz x19, 1f 558 559 /* Create the mapping for FDT data (2 MiB max) */ 560 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 561 ldr x9, =(L2_SIZE) 562 add x16, x16, x9 /* VA start */ 563 mov x8, x16 564 mov x9, x0 /* PA start */ 565 /* Update the module pointer to point at the allocated memory */ 566 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 567 add x0, x0, x8 /* Add the aligned virtual address */ 568 569 mov x10, #1 570 bl build_l2_block_pagetable 571 5721: 573#endif 574 575 /* Move to the l1 table */ 576 add x27, x27, #PAGE_SIZE 577 578 /* Link the l1 -> l2 table */ 579 mov x9, x6 580 mov x6, x27 581 bl link_l1_pagetable 582 583 /* Move to the l0 table */ 584 add x27, x27, #PAGE_SIZE 585 586 /* Link the l0 -> l1 table */ 587 mov x9, x6 588 mov x6, x27 589 mov x10, #1 590 bl link_l0_pagetable 591 592 /* Restore the Link register */ 593 mov x30, x5 594 ret 595LEND(create_pagetables) 596 597/* 598 * Builds an L0 -> L1 table descriptor 599 * 600 * x6 = L0 table 601 * x8 = Virtual Address 602 * x9 = L1 PA (trashed) 603 * x10 = Entry count (trashed) 604 * x11, x12 and x13 are trashed 605 */ 606LENTRY(link_l0_pagetable) 607 /* 608 * Link an L0 -> L1 table entry. 609 */ 610 /* Find the table index */ 611 lsr x11, x8, #L0_SHIFT 612 and x11, x11, #L0_ADDR_MASK 613 614 /* Build the L0 block entry */ 615 mov x12, #L0_TABLE 616 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 617 618 /* Only use the output address bits */ 619 lsr x9, x9, #PAGE_SHIFT 6201: orr x13, x12, x9, lsl #PAGE_SHIFT 621 622 /* Store the entry */ 623 str x13, [x6, x11, lsl #3] 624 625 sub x10, x10, #1 626 add x11, x11, #1 627 add x9, x9, #1 628 cbnz x10, 1b 629 630 ret 631LEND(link_l0_pagetable) 632 633/* 634 * Builds an L1 -> L2 table descriptor 635 * 636 * x6 = L1 table 637 * x8 = Virtual Address 638 * x9 = L2 PA (trashed) 639 * x11, x12 and x13 are trashed 640 */ 641LENTRY(link_l1_pagetable) 642 /* 643 * Link an L1 -> L2 table entry. 644 */ 645 /* Find the table index */ 646 lsr x11, x8, #L1_SHIFT 647 and x11, x11, #Ln_ADDR_MASK 648 649 /* Build the L1 block entry */ 650 mov x12, #L1_TABLE 651 652 /* Only use the output address bits */ 653 lsr x9, x9, #PAGE_SHIFT 654 orr x13, x12, x9, lsl #PAGE_SHIFT 655 656 /* Store the entry */ 657 str x13, [x6, x11, lsl #3] 658 659 ret 660LEND(link_l1_pagetable) 661 662/* 663 * Builds count 2 MiB page table entry 664 * x6 = L2 table 665 * x7 = Block attributes 666 * x8 = VA start 667 * x9 = PA start (trashed) 668 * x10 = Entry count (trashed) 669 * x11, x12 and x13 are trashed 670 */ 671LENTRY(build_l2_block_pagetable) 672 /* 673 * Build the L2 table entry. 674 */ 675 /* Find the table index */ 676 lsr x11, x8, #L2_SHIFT 677 and x11, x11, #Ln_ADDR_MASK 678 679 /* Build the L2 block entry */ 680 orr x12, x7, #L2_BLOCK 681 orr x12, x12, #(ATTR_DEFAULT) 682 orr x12, x12, #(ATTR_S1_UXN) 683 684 /* Only use the output address bits */ 685 lsr x9, x9, #L2_SHIFT 686 687 /* Set the physical address for this virtual address */ 6881: orr x13, x12, x9, lsl #L2_SHIFT 689 690 /* Store the entry */ 691 str x13, [x6, x11, lsl #3] 692 693 sub x10, x10, #1 694 add x11, x11, #1 695 add x9, x9, #1 696 cbnz x10, 1b 697 698 ret 699LEND(build_l2_block_pagetable) 700 701#if PAGE_SIZE != PAGE_SIZE_4K 702/* 703 * Builds an L2 -> L3 table descriptor 704 * 705 * x6 = L2 table 706 * x8 = Virtual Address 707 * x9 = L3 PA (trashed) 708 * x11, x12 and x13 are trashed 709 */ 710LENTRY(link_l2_pagetable) 711 /* 712 * Link an L2 -> L3 table entry. 713 */ 714 /* Find the table index */ 715 lsr x11, x8, #L2_SHIFT 716 and x11, x11, #Ln_ADDR_MASK 717 718 /* Build the L1 block entry */ 719 mov x12, #L2_TABLE 720 721 /* Only use the output address bits */ 722 lsr x9, x9, #PAGE_SHIFT 723 orr x13, x12, x9, lsl #PAGE_SHIFT 724 725 /* Store the entry */ 726 str x13, [x6, x11, lsl #3] 727 728 ret 729LEND(link_l2_pagetable) 730 731/* 732 * Builds count level 3 page table entries 733 * x6 = L3 table 734 * x7 = Block attributes 735 * x8 = VA start 736 * x9 = PA start (trashed) 737 * x10 = Entry count (trashed) 738 * x11, x12 and x13 are trashed 739 */ 740LENTRY(build_l3_page_pagetable) 741 /* 742 * Build the L3 table entry. 743 */ 744 /* Find the table index */ 745 lsr x11, x8, #L3_SHIFT 746 and x11, x11, #Ln_ADDR_MASK 747 748 /* Build the L3 page entry */ 749 orr x12, x7, #L3_PAGE 750 orr x12, x12, #(ATTR_DEFAULT) 751 orr x12, x12, #(ATTR_S1_UXN) 752 753 /* Only use the output address bits */ 754 lsr x9, x9, #L3_SHIFT 755 756 /* Set the physical address for this virtual address */ 7571: orr x13, x12, x9, lsl #L3_SHIFT 758 759 /* Store the entry */ 760 str x13, [x6, x11, lsl #3] 761 762 sub x10, x10, #1 763 add x11, x11, #1 764 add x9, x9, #1 765 cbnz x10, 1b 766 767 ret 768LEND(build_l3_page_pagetable) 769#endif 770 771LENTRY(start_mmu) 772 dsb sy 773 774 /* Load the exception vectors */ 775 ldr x2, =exception_vectors 776 msr vbar_el1, x2 777 778 /* Load ttbr0 and ttbr1 */ 779 msr ttbr0_el1, x27 780 msr ttbr1_el1, x24 781 isb 782 783 /* Clear the Monitor Debug System control register */ 784 msr mdscr_el1, xzr 785 786 /* Invalidate the TLB */ 787 tlbi vmalle1is 788 dsb ish 789 isb 790 791 ldr x2, mair 792 msr mair_el1, x2 793 794 /* 795 * Setup TCR according to the PARange and ASIDBits fields 796 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 797 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 798 * to 1 only if the ASIDBits field equals 0b0010. 799 */ 800 ldr x2, tcr 801 mrs x3, id_aa64mmfr0_el1 802 803 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 804 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 805 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 806 807 /* Check if the HW supports 16 bit ASIDS */ 808 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 809 /* If so x3 == 1, else x3 == 0 */ 810 cset x3, eq 811 /* Set TCR.AS with x3 */ 812 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 813 814 /* 815 * Check if the HW supports access flag and dirty state updates, 816 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 817 */ 818 mrs x3, id_aa64mmfr1_el1 819 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 820 cmp x3, #1 821 b.ne 1f 822 orr x2, x2, #(TCR_HA) 823 b 2f 8241: 825 cmp x3, #2 826 b.ne 2f 827 orr x2, x2, #(TCR_HA | TCR_HD) 8282: 829 msr tcr_el1, x2 830 831 /* 832 * Setup SCTLR. 833 */ 834 ldr x2, sctlr_set 835 ldr x3, sctlr_clear 836 mrs x1, sctlr_el1 837 bic x1, x1, x3 /* Clear the required bits */ 838 orr x1, x1, x2 /* Set the required bits */ 839 msr sctlr_el1, x1 840 isb 841 842 ret 843 844 .align 3 845mair: 846 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 847 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 848 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 849 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 850 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 851tcr: 852#if PAGE_SIZE == PAGE_SIZE_4K 853#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 854#elif PAGE_SIZE == PAGE_SIZE_16K 855#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 856#else 857#error Unsupported page size 858#endif 859 860 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 861 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 862sctlr_set: 863 /* Bits to set */ 864 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 865 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 866 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 867 SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 868sctlr_clear: 869 /* Bits to clear */ 870 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 871 SCTLR_ITD | SCTLR_A) 872LEND(start_mmu) 873 874ENTRY(abort) 875 b abort 876END(abort) 877 878.bss 879 .align PAGE_SHIFT 880initstack: 881 .space (PAGE_SIZE * KSTACK_PAGES) 882initstack_end: 883 884 .section .init_pagetable, "aw", %nobits 885 .align PAGE_SHIFT 886 /* 887 * 6 initial tables (in the following order): 888 * L2 for kernel (High addresses) 889 * L1 for kernel 890 * L0 for kernel 891 * L1 bootstrap for user (Low addresses) 892 * L0 bootstrap for user 893 * L0 for user 894 */ 895 .globl pagetable_l0_ttbr1 896pagetable: 897#if PAGE_SIZE != PAGE_SIZE_4K 898 .space (PAGE_SIZE * L3_PAGE_COUNT) 899pagetable_l2_ttbr1: 900#endif 901 .space PAGE_SIZE 902pagetable_l1_ttbr1: 903 .space PAGE_SIZE 904pagetable_l0_ttbr1: 905 .space PAGE_SIZE 906pagetable_l2_ttbr0_bootstrap: 907 .space PAGE_SIZE 908pagetable_l1_ttbr0_bootstrap: 909 .space PAGE_SIZE 910pagetable_l0_ttbr0_boostrap: 911 .space PAGE_SIZE 912pagetable_l0_ttbr0: 913 .space PAGE_SIZE 914pagetable_end: 915 916el2_pagetable: 917 .space PAGE_SIZE 918 919 .section .rodata, "a", %progbits 920 .globl aarch32_sigcode 921 .align 2 922aarch32_sigcode: 923 .word 0xe1a0000d // mov r0, sp 924 .word 0xe2800040 // add r0, r0, #SIGF_UC 925 .word 0xe59f700c // ldr r7, [pc, #12] 926 .word 0xef000000 // swi #0 927 .word 0xe59f7008 // ldr r7, [pc, #8] 928 .word 0xef000000 // swi #0 929 .word 0xeafffffa // b . - 16 930 .word SYS_sigreturn 931 .word SYS_exit 932 .align 3 933 .size aarch32_sigcode, . - aarch32_sigcode 934aarch32_esigcode: 935 .data 936 .global sz_aarch32_sigcode 937sz_aarch32_sigcode: 938 .quad aarch32_esigcode - aarch32_sigcode 939