1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42#if PAGE_SIZE == PAGE_SIZE_16K 43/* 44 * The number of level 3 tables to create. 32 will allow for 1G of address 45 * space, the same as a single level 2 page with 4k pages. 46 */ 47#define L3_PAGE_COUNT 32 48#endif 49 50 .globl kernbase 51 .set kernbase, KERNBASE 52 53/* 54 * We assume: 55 * MMU on with an identity map, or off 56 * D-Cache: off 57 * I-Cache: on or off 58 * We are loaded at a 2MiB aligned address 59 */ 60 61ENTRY(_start) 62 /* Drop to EL1 */ 63 bl drop_to_el1 64 65 /* 66 * Disable the MMU. We may have entered the kernel with it on and 67 * will need to update the tables later. If this has been set up 68 * with anything other than a VA == PA map then this will fail, 69 * but in this case the code to find where we are running from 70 * would have also failed. 71 */ 72 dsb sy 73 mrs x2, sctlr_el1 74 bic x2, x2, SCTLR_M 75 msr sctlr_el1, x2 76 isb 77 78 /* Set the context id */ 79 msr contextidr_el1, xzr 80 81 /* Get the virt -> phys offset */ 82 bl get_virt_delta 83 84 /* 85 * At this point: 86 * x29 = PA - VA 87 * x28 = Our physical load address 88 */ 89 90 /* Create the page tables */ 91 bl create_pagetables 92 93 /* 94 * At this point: 95 * x27 = TTBR0 table 96 * x26 = Kernel L1 table 97 * x24 = TTBR1 table 98 */ 99 100 /* Enable the mmu */ 101 bl start_mmu 102 103 /* Load the new ttbr0 pagetable */ 104 adrp x27, pagetable_l0_ttbr0 105 add x27, x27, :lo12:pagetable_l0_ttbr0 106 107 /* Jump to the virtual address space */ 108 ldr x15, .Lvirtdone 109 br x15 110 111virtdone: 112 /* Set up the stack */ 113 adrp x25, initstack_end 114 add x25, x25, :lo12:initstack_end 115 mov sp, x25 116 sub sp, sp, #PCB_SIZE 117 118 /* Zero the BSS */ 119 ldr x15, .Lbss 120 ldr x14, .Lend 1211: 122 str xzr, [x15], #8 123 cmp x15, x14 124 b.lo 1b 125 126#if defined(PERTHREAD_SSP) 127 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 128 adrp x15, boot_canary 129 add x15, x15, :lo12:boot_canary 130 msr sp_el0, x15 131#endif 132 133 /* Backup the module pointer */ 134 mov x1, x0 135 136 sub sp, sp, #BOOTPARAMS_SIZE 137 mov x0, sp 138 139 /* Degate the delda so it is VA -> PA */ 140 neg x29, x29 141 142 str x1, [x0, #BP_MODULEP] 143 str x29, [x0, #BP_KERN_DELTA] 144 adrp x25, initstack 145 add x25, x25, :lo12:initstack 146 str x25, [x0, #BP_KERN_STACK] 147 str x27, [x0, #BP_KERN_TTBR0] 148 str x23, [x0, #BP_BOOT_EL] 149 150 /* trace back starts here */ 151 mov fp, #0 152 /* Branch to C code */ 153 bl initarm 154 /* We are done with the boot params */ 155 add sp, sp, #BOOTPARAMS_SIZE 156 157 /* 158 * Enable pointer authentication in the kernel. We set the keys for 159 * thread0 in initarm so have to wait until it returns to enable it. 160 * If we were to enable it in initarm then any authentication when 161 * returning would fail as it was called with pointer authentication 162 * disabled. 163 */ 164 bl ptrauth_start 165 166 bl mi_startup 167 168 /* We should not get here */ 169 brk 0 170 171 .align 3 172.Lvirtdone: 173 .quad virtdone 174.Lbss: 175 .quad __bss_start 176.Lend: 177 .quad __bss_end 178END(_start) 179 180#ifdef SMP 181/* 182 * mpentry(unsigned long) 183 * 184 * Called by a core when it is being brought online. 185 * The data in x0 is passed straight to init_secondary. 186 */ 187ENTRY(mpentry) 188 /* Disable interrupts */ 189 msr daifset, #DAIF_INTR 190 191 /* Drop to EL1 */ 192 bl drop_to_el1 193 194 /* Set the context id */ 195 msr contextidr_el1, xzr 196 197 /* Load the kernel page table */ 198 adrp x24, pagetable_l0_ttbr1 199 add x24, x24, :lo12:pagetable_l0_ttbr1 200 /* Load the identity page table */ 201 adrp x27, pagetable_l0_ttbr0_boostrap 202 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 203 204 /* Enable the mmu */ 205 bl start_mmu 206 207 /* Load the new ttbr0 pagetable */ 208 adrp x27, pagetable_l0_ttbr0 209 add x27, x27, :lo12:pagetable_l0_ttbr0 210 211 /* Jump to the virtual address space */ 212 ldr x15, =mp_virtdone 213 br x15 214 215mp_virtdone: 216 /* Start using the AP boot stack */ 217 ldr x4, =bootstack 218 ldr x4, [x4] 219 mov sp, x4 220 221#if defined(PERTHREAD_SSP) 222 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 223 adrp x15, boot_canary 224 add x15, x15, :lo12:boot_canary 225 msr sp_el0, x15 226#endif 227 228 /* Load the kernel ttbr0 pagetable */ 229 msr ttbr0_el1, x27 230 isb 231 232 /* Invalidate the TLB */ 233 tlbi vmalle1 234 dsb sy 235 isb 236 237 b init_secondary 238END(mpentry) 239#endif 240 241/* 242 * If we are started in EL2, configure the required hypervisor 243 * registers and drop to EL1. 244 */ 245LENTRY(drop_to_el1) 246 mrs x23, CurrentEL 247 lsr x23, x23, #2 248 cmp x23, #0x2 249 b.eq 1f 250 ret 2511: 252 /* 253 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 254 * which may break address translation. 255 */ 256 dsb sy 257 mrs x2, sctlr_el2 258 bic x2, x2, SCTLR_M 259 msr sctlr_el2, x2 260 isb 261 262 /* Configure the Hypervisor */ 263 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 264 msr hcr_el2, x2 265 266 /* Stash value of HCR_EL2 for later */ 267 isb 268 mrs x4, hcr_el2 269 270 /* Load the Virtualization Process ID Register */ 271 mrs x2, midr_el1 272 msr vpidr_el2, x2 273 274 /* Load the Virtualization Multiprocess ID Register */ 275 mrs x2, mpidr_el1 276 msr vmpidr_el2, x2 277 278 /* Set the bits that need to be 1 in sctlr_el1 */ 279 ldr x2, .Lsctlr_res1 280 msr sctlr_el1, x2 281 282 /* 283 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 284 * don't trap to EL2 for SIMD register usage to have at least a 285 * minimally usable system. 286 */ 287 tst x4, #HCR_E2H 288 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 289 mov x4, #CPTR_FPEN /* HCR_E2H == 1 */ 290 csel x2, x3, x4, eq 291 msr cptr_el2, x2 292 293 /* Don't trap to EL2 for CP15 traps */ 294 msr hstr_el2, xzr 295 296 /* Enable access to the physical timers at EL1 */ 297 mrs x2, cnthctl_el2 298 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 299 msr cnthctl_el2, x2 300 301 /* Set the counter offset to a known value */ 302 msr cntvoff_el2, xzr 303 304 /* Hypervisor trap functions */ 305 adrp x2, hyp_stub_vectors 306 add x2, x2, :lo12:hyp_stub_vectors 307 msr vbar_el2, x2 308 309 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 310 msr vttbr_el2, xzr 311 312 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 313 msr spsr_el2, x2 314 315 /* Configure GICv3 CPU interface */ 316 mrs x2, id_aa64pfr0_el1 317 /* Extract GIC bits from the register */ 318 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 319 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 320 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 321 b.ne 2f 322 323 mrs x2, icc_sre_el2 324 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 325 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 326 msr icc_sre_el2, x2 3272: 328 329 /* Set the address to return to our return address */ 330 msr elr_el2, x30 331 isb 332 333 eret 334 335 .align 3 336.Lsctlr_res1: 337 .quad SCTLR_RES1 338LEND(drop_to_el1) 339 340/* 341 * Get the delta between the physical address we were loaded to and the 342 * virtual address we expect to run from. This is used when building the 343 * initial page table. 344 */ 345LENTRY(get_virt_delta) 346 /* Load the physical address of virt_map */ 347 adrp x29, virt_map 348 add x29, x29, :lo12:virt_map 349 /* Load the virtual address of virt_map stored in virt_map */ 350 ldr x28, [x29] 351 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 352 sub x29, x29, x28 353 /* Find the load address for the kernel */ 354 mov x28, #(KERNBASE) 355 add x28, x28, x29 356 ret 357 358 .align 3 359virt_map: 360 .quad virt_map 361LEND(get_virt_delta) 362 363/* 364 * This builds the page tables containing the identity map, and the kernel 365 * virtual map. 366 * 367 * It relys on: 368 * We were loaded to an address that is on a 2MiB boundary 369 * All the memory must not cross a 1GiB boundaty 370 * x28 contains the physical address we were loaded from 371 * 372 * TODO: This is out of date. 373 * There are at least 5 pages before that address for the page tables 374 * The pages used are: 375 * - The Kernel L2 table 376 * - The Kernel L1 table 377 * - The Kernel L0 table (TTBR1) 378 * - The identity (PA = VA) L1 table 379 * - The identity (PA = VA) L0 table (TTBR0) 380 */ 381LENTRY(create_pagetables) 382 /* Save the Link register */ 383 mov x5, x30 384 385 /* Clean the page table */ 386 adrp x6, pagetable 387 add x6, x6, :lo12:pagetable 388 mov x26, x6 389 adrp x27, pagetable_end 390 add x27, x27, :lo12:pagetable_end 3911: 392 stp xzr, xzr, [x6], #16 393 stp xzr, xzr, [x6], #16 394 stp xzr, xzr, [x6], #16 395 stp xzr, xzr, [x6], #16 396 cmp x6, x27 397 b.lo 1b 398 399 /* 400 * Build the TTBR1 maps. 401 */ 402 403 /* Find the size of the kernel */ 404 mov x6, #(KERNBASE) 405 406#if defined(LINUX_BOOT_ABI) 407 /* X19 is used as 'map FDT data' flag */ 408 mov x19, xzr 409 410 /* No modules or FDT pointer ? */ 411 cbz x0, booti_no_fdt 412 413 /* 414 * Test if x0 points to modules descriptor(virtual address) or 415 * to FDT (physical address) 416 */ 417 cmp x0, x6 /* x6 is #(KERNBASE) */ 418 b.lo booti_fdt 419#endif 420 421 /* Booted with modules pointer */ 422 /* Find modulep - begin */ 423 sub x8, x0, x6 424 /* 425 * Add space for the module data. When PAGE_SIZE is 4k this will 426 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 427 * larger it will be at least as large as we use smaller level 3 428 * pages. 429 */ 430 ldr x7, =((6 * 1024 * 1024) - 1) 431 add x8, x8, x7 432 b common 433 434#if defined(LINUX_BOOT_ABI) 435booti_fdt: 436 /* Booted by U-Boot booti with FDT data */ 437 /* Set 'map FDT data' flag */ 438 mov x19, #1 439 440booti_no_fdt: 441 /* Booted by U-Boot booti without FTD data */ 442 /* Find the end - begin */ 443 ldr x7, .Lend 444 sub x8, x7, x6 445 446 /* 447 * Add one 2MiB page for copy of FDT data (maximum FDT size), 448 * one for metadata and round up 449 */ 450 ldr x7, =(3 * L2_SIZE - 1) 451 add x8, x8, x7 452#endif 453 454common: 455#if PAGE_SIZE != PAGE_SIZE_4K 456 /* 457 * Create L3 pages. The kernel will be loaded at a 2M aligned 458 * address, however L2 blocks are too large when the page size is 459 * not 4k to map the kernel with such an aligned address. However, 460 * when the page size is larger than 4k, L2 blocks are too large to 461 * map the kernel with such an alignment. 462 */ 463 464 /* Get the number of l3 pages to allocate, rounded down */ 465 lsr x10, x8, #(L3_SHIFT) 466 467 /* Create the kernel space L2 table */ 468 mov x6, x26 469 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 470 mov x8, #(KERNBASE) 471 mov x9, x28 472 bl build_l3_page_pagetable 473 474 /* Move to the l2 table */ 475 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 476 add x26, x26, x9 477 478 /* Link the l2 -> l3 table */ 479 mov x9, x6 480 mov x6, x26 481 bl link_l2_pagetable 482#else 483 /* Get the number of l2 pages to allocate, rounded down */ 484 lsr x10, x8, #(L2_SHIFT) 485 486 /* Create the kernel space L2 table */ 487 mov x6, x26 488 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 489 mov x8, #(KERNBASE) 490 mov x9, x28 491 bl build_l2_block_pagetable 492#endif 493 494 /* Move to the l1 table */ 495 add x26, x26, #PAGE_SIZE 496 497 /* Link the l1 -> l2 table */ 498 mov x9, x6 499 mov x6, x26 500 bl link_l1_pagetable 501 502 /* Move to the l0 table */ 503 add x24, x26, #PAGE_SIZE 504 505 /* Link the l0 -> l1 table */ 506 mov x9, x6 507 mov x6, x24 508 mov x10, #1 509 bl link_l0_pagetable 510 511 /* 512 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 513 * They are only needed early on, so the VA = PA map is uncached. 514 */ 515 add x27, x24, #PAGE_SIZE 516 517 mov x6, x27 /* The initial page table */ 518 519 /* Create the VA = PA map */ 520 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 521 adrp x16, _start 522 and x16, x16, #(~L2_OFFSET) 523 mov x9, x16 /* PA start */ 524 mov x8, x16 /* VA start (== PA start) */ 525 mov x10, #1 526 bl build_l2_block_pagetable 527 528#if defined(SOCDEV_PA) 529 /* Create a table for the UART */ 530 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 531 ldr x9, =(L2_SIZE) 532 add x16, x16, x9 /* VA start */ 533 mov x8, x16 534 535 /* Store the socdev virtual address */ 536 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 537 adrp x9, socdev_va 538 str x17, [x9, :lo12:socdev_va] 539 540 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 541 mov x10, #1 542 bl build_l2_block_pagetable 543#endif 544 545#if defined(LINUX_BOOT_ABI) 546 /* Map FDT data ? */ 547 cbz x19, 1f 548 549 /* Create the mapping for FDT data (2 MiB max) */ 550 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 551 ldr x9, =(L2_SIZE) 552 add x16, x16, x9 /* VA start */ 553 mov x8, x16 554 mov x9, x0 /* PA start */ 555 /* Update the module pointer to point at the allocated memory */ 556 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 557 add x0, x0, x8 /* Add the aligned virtual address */ 558 559 mov x10, #1 560 bl build_l2_block_pagetable 561 5621: 563#endif 564 565 /* Move to the l1 table */ 566 add x27, x27, #PAGE_SIZE 567 568 /* Link the l1 -> l2 table */ 569 mov x9, x6 570 mov x6, x27 571 bl link_l1_pagetable 572 573 /* Move to the l0 table */ 574 add x27, x27, #PAGE_SIZE 575 576 /* Link the l0 -> l1 table */ 577 mov x9, x6 578 mov x6, x27 579 mov x10, #1 580 bl link_l0_pagetable 581 582 /* Restore the Link register */ 583 mov x30, x5 584 ret 585LEND(create_pagetables) 586 587/* 588 * Builds an L0 -> L1 table descriptor 589 * 590 * x6 = L0 table 591 * x8 = Virtual Address 592 * x9 = L1 PA (trashed) 593 * x10 = Entry count (trashed) 594 * x11, x12 and x13 are trashed 595 */ 596LENTRY(link_l0_pagetable) 597 /* 598 * Link an L0 -> L1 table entry. 599 */ 600 /* Find the table index */ 601 lsr x11, x8, #L0_SHIFT 602 and x11, x11, #L0_ADDR_MASK 603 604 /* Build the L0 block entry */ 605 mov x12, #L0_TABLE 606 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 607 608 /* Only use the output address bits */ 609 lsr x9, x9, #PAGE_SHIFT 6101: orr x13, x12, x9, lsl #PAGE_SHIFT 611 612 /* Store the entry */ 613 str x13, [x6, x11, lsl #3] 614 615 sub x10, x10, #1 616 add x11, x11, #1 617 add x9, x9, #1 618 cbnz x10, 1b 619 620 ret 621LEND(link_l0_pagetable) 622 623/* 624 * Builds an L1 -> L2 table descriptor 625 * 626 * x6 = L1 table 627 * x8 = Virtual Address 628 * x9 = L2 PA (trashed) 629 * x11, x12 and x13 are trashed 630 */ 631LENTRY(link_l1_pagetable) 632 /* 633 * Link an L1 -> L2 table entry. 634 */ 635 /* Find the table index */ 636 lsr x11, x8, #L1_SHIFT 637 and x11, x11, #Ln_ADDR_MASK 638 639 /* Build the L1 block entry */ 640 mov x12, #L1_TABLE 641 642 /* Only use the output address bits */ 643 lsr x9, x9, #PAGE_SHIFT 644 orr x13, x12, x9, lsl #PAGE_SHIFT 645 646 /* Store the entry */ 647 str x13, [x6, x11, lsl #3] 648 649 ret 650LEND(link_l1_pagetable) 651 652/* 653 * Builds count 2 MiB page table entry 654 * x6 = L2 table 655 * x7 = Block attributes 656 * x8 = VA start 657 * x9 = PA start (trashed) 658 * x10 = Entry count (trashed) 659 * x11, x12 and x13 are trashed 660 */ 661LENTRY(build_l2_block_pagetable) 662 /* 663 * Build the L2 table entry. 664 */ 665 /* Find the table index */ 666 lsr x11, x8, #L2_SHIFT 667 and x11, x11, #Ln_ADDR_MASK 668 669 /* Build the L2 block entry */ 670 orr x12, x7, #L2_BLOCK 671 orr x12, x12, #(ATTR_DEFAULT) 672 orr x12, x12, #(ATTR_S1_UXN) 673 674 /* Only use the output address bits */ 675 lsr x9, x9, #L2_SHIFT 676 677 /* Set the physical address for this virtual address */ 6781: orr x13, x12, x9, lsl #L2_SHIFT 679 680 /* Store the entry */ 681 str x13, [x6, x11, lsl #3] 682 683 sub x10, x10, #1 684 add x11, x11, #1 685 add x9, x9, #1 686 cbnz x10, 1b 687 688 ret 689LEND(build_l2_block_pagetable) 690 691#if PAGE_SIZE != PAGE_SIZE_4K 692/* 693 * Builds an L2 -> L3 table descriptor 694 * 695 * x6 = L2 table 696 * x8 = Virtual Address 697 * x9 = L3 PA (trashed) 698 * x11, x12 and x13 are trashed 699 */ 700LENTRY(link_l2_pagetable) 701 /* 702 * Link an L2 -> L3 table entry. 703 */ 704 /* Find the table index */ 705 lsr x11, x8, #L2_SHIFT 706 and x11, x11, #Ln_ADDR_MASK 707 708 /* Build the L1 block entry */ 709 mov x12, #L2_TABLE 710 711 /* Only use the output address bits */ 712 lsr x9, x9, #PAGE_SHIFT 713 orr x13, x12, x9, lsl #PAGE_SHIFT 714 715 /* Store the entry */ 716 str x13, [x6, x11, lsl #3] 717 718 ret 719LEND(link_l2_pagetable) 720 721/* 722 * Builds count level 3 page table entries 723 * x6 = L3 table 724 * x7 = Block attributes 725 * x8 = VA start 726 * x9 = PA start (trashed) 727 * x10 = Entry count (trashed) 728 * x11, x12 and x13 are trashed 729 */ 730LENTRY(build_l3_page_pagetable) 731 /* 732 * Build the L3 table entry. 733 */ 734 /* Find the table index */ 735 lsr x11, x8, #L3_SHIFT 736 and x11, x11, #Ln_ADDR_MASK 737 738 /* Build the L3 page entry */ 739 orr x12, x7, #L3_PAGE 740 orr x12, x12, #(ATTR_DEFAULT) 741 orr x12, x12, #(ATTR_S1_UXN) 742 743 /* Only use the output address bits */ 744 lsr x9, x9, #L3_SHIFT 745 746 /* Set the physical address for this virtual address */ 7471: orr x13, x12, x9, lsl #L3_SHIFT 748 749 /* Store the entry */ 750 str x13, [x6, x11, lsl #3] 751 752 sub x10, x10, #1 753 add x11, x11, #1 754 add x9, x9, #1 755 cbnz x10, 1b 756 757 ret 758LEND(build_l3_page_pagetable) 759#endif 760 761LENTRY(start_mmu) 762 dsb sy 763 764 /* Load the exception vectors */ 765 ldr x2, =exception_vectors 766 msr vbar_el1, x2 767 768 /* Load ttbr0 and ttbr1 */ 769 msr ttbr0_el1, x27 770 msr ttbr1_el1, x24 771 isb 772 773 /* Clear the Monitor Debug System control register */ 774 msr mdscr_el1, xzr 775 776 /* Invalidate the TLB */ 777 tlbi vmalle1is 778 dsb ish 779 isb 780 781 ldr x2, mair 782 msr mair_el1, x2 783 784 /* 785 * Setup TCR according to the PARange and ASIDBits fields 786 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 787 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 788 * to 1 only if the ASIDBits field equals 0b0010. 789 */ 790 ldr x2, tcr 791 mrs x3, id_aa64mmfr0_el1 792 793 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 794 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 795 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 796 797 /* Check if the HW supports 16 bit ASIDS */ 798 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 799 /* If so x3 == 1, else x3 == 0 */ 800 cset x3, eq 801 /* Set TCR.AS with x3 */ 802 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 803 804 /* 805 * Check if the HW supports access flag and dirty state updates, 806 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 807 */ 808 mrs x3, id_aa64mmfr1_el1 809 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 810 cmp x3, #1 811 b.ne 1f 812 orr x2, x2, #(TCR_HA) 813 b 2f 8141: 815 cmp x3, #2 816 b.ne 2f 817 orr x2, x2, #(TCR_HA | TCR_HD) 8182: 819 msr tcr_el1, x2 820 821 /* 822 * Setup SCTLR. 823 */ 824 ldr x2, sctlr_set 825 ldr x3, sctlr_clear 826 mrs x1, sctlr_el1 827 bic x1, x1, x3 /* Clear the required bits */ 828 orr x1, x1, x2 /* Set the required bits */ 829 msr sctlr_el1, x1 830 isb 831 832 ret 833 834 .align 3 835mair: 836 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 837 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 838 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 839 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 840 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 841tcr: 842#if PAGE_SIZE == PAGE_SIZE_4K 843#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 844#elif PAGE_SIZE == PAGE_SIZE_16K 845#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 846#else 847#error Unsupported page size 848#endif 849 850 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 851 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 852sctlr_set: 853 /* Bits to set */ 854 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 855 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 856 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 857 SCTLR_M | SCTLR_CP15BEN) 858sctlr_clear: 859 /* Bits to clear */ 860 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 861 SCTLR_ITD | SCTLR_A) 862LEND(start_mmu) 863 864ENTRY(abort) 865 b abort 866END(abort) 867 868 .section .init_pagetable, "aw", %nobits 869 .align PAGE_SHIFT 870 /* 871 * 6 initial tables (in the following order): 872 * L2 for kernel (High addresses) 873 * L1 for kernel 874 * L0 for kernel 875 * L1 bootstrap for user (Low addresses) 876 * L0 bootstrap for user 877 * L0 for user 878 */ 879 .globl pagetable_l0_ttbr1 880pagetable: 881#if PAGE_SIZE != PAGE_SIZE_4K 882 .space (PAGE_SIZE * L3_PAGE_COUNT) 883pagetable_l2_ttbr1: 884#endif 885 .space PAGE_SIZE 886pagetable_l1_ttbr1: 887 .space PAGE_SIZE 888pagetable_l0_ttbr1: 889 .space PAGE_SIZE 890pagetable_l2_ttbr0_bootstrap: 891 .space PAGE_SIZE 892pagetable_l1_ttbr0_bootstrap: 893 .space PAGE_SIZE 894pagetable_l0_ttbr0_boostrap: 895 .space PAGE_SIZE 896pagetable_l0_ttbr0: 897 .space PAGE_SIZE 898 899pagetable_end: 900 901el2_pagetable: 902 .space PAGE_SIZE 903 904 .align 4 905initstack: 906 .space (PAGE_SIZE * KSTACK_PAGES) 907initstack_end: 908 909 910.text 911EENTRY(aarch32_sigcode) 912 .word 0xe1a0000d // mov r0, sp 913 .word 0xe2800040 // add r0, r0, #SIGF_UC 914 .word 0xe59f700c // ldr r7, [pc, #12] 915 .word 0xef000000 // swi #0 916 .word 0xe59f7008 // ldr r7, [pc, #8] 917 .word 0xef000000 // swi #0 918 .word 0xeafffffa // b . - 16 919EEND(aarch32_sigcode) 920 .word SYS_sigreturn 921 .word SYS_exit 922 .align 3 923aarch32_esigcode: 924 .data 925 .global sz_aarch32_sigcode 926sz_aarch32_sigcode: 927 .quad aarch32_esigcode - aarch32_sigcode 928