1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42#if PAGE_SIZE == PAGE_SIZE_16K 43/* 44 * The number of level 3 tables to create. 32 will allow for 1G of address 45 * space, the same as a single level 2 page with 4k pages. 46 */ 47#define L3_PAGE_COUNT 32 48#endif 49 50 .globl kernbase 51 .set kernbase, KERNBASE 52 53/* 54 * We assume: 55 * MMU on with an identity map, or off 56 * D-Cache: off 57 * I-Cache: on or off 58 * We are loaded at a 2MiB aligned address 59 */ 60 61ENTRY(_start) 62 /* Drop to EL1 */ 63 bl drop_to_el1 64 65 /* 66 * Disable the MMU. We may have entered the kernel with it on and 67 * will need to update the tables later. If this has been set up 68 * with anything other than a VA == PA map then this will fail, 69 * but in this case the code to find where we are running from 70 * would have also failed. 71 */ 72 dsb sy 73 mrs x2, sctlr_el1 74 bic x2, x2, SCTLR_M 75 msr sctlr_el1, x2 76 isb 77 78 /* Set the context id */ 79 msr contextidr_el1, xzr 80 81 /* Get the virt -> phys offset */ 82 bl get_virt_delta 83 84 /* 85 * At this point: 86 * x29 = PA - VA 87 * x28 = Our physical load address 88 */ 89 90 /* Create the page tables */ 91 bl create_pagetables 92 93 /* 94 * At this point: 95 * x27 = TTBR0 table 96 * x26 = Kernel L1 table 97 * x24 = TTBR1 table 98 */ 99 100 /* Enable the mmu */ 101 bl start_mmu 102 103 /* Load the new ttbr0 pagetable */ 104 adrp x27, pagetable_l0_ttbr0 105 add x27, x27, :lo12:pagetable_l0_ttbr0 106 107 /* Jump to the virtual address space */ 108 ldr x15, .Lvirtdone 109 br x15 110 111virtdone: 112 /* Set up the stack */ 113 adrp x25, initstack_end 114 add x25, x25, :lo12:initstack_end 115 mov sp, x25 116 sub sp, sp, #PCB_SIZE 117 118 /* Zero the BSS */ 119 ldr x15, .Lbss 120 ldr x14, .Lend 1211: 122 str xzr, [x15], #8 123 cmp x15, x14 124 b.lo 1b 125 126#if defined(PERTHREAD_SSP) 127 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 128 adrp x15, boot_canary 129 add x15, x15, :lo12:boot_canary 130 msr sp_el0, x15 131#endif 132 133 /* Backup the module pointer */ 134 mov x1, x0 135 136 sub sp, sp, #BOOTPARAMS_SIZE 137 mov x0, sp 138 139 /* Degate the delda so it is VA -> PA */ 140 neg x29, x29 141 142 str x1, [x0, #BP_MODULEP] 143 str x29, [x0, #BP_KERN_DELTA] 144 adrp x25, initstack 145 add x25, x25, :lo12:initstack 146 str x25, [x0, #BP_KERN_STACK] 147 str x27, [x0, #BP_KERN_TTBR0] 148 str x23, [x0, #BP_BOOT_EL] 149 str x4, [x0, #BP_HCR_EL2] 150 151 /* trace back starts here */ 152 mov fp, #0 153 /* Branch to C code */ 154 bl initarm 155 /* We are done with the boot params */ 156 add sp, sp, #BOOTPARAMS_SIZE 157 158 /* 159 * Enable pointer authentication in the kernel. We set the keys for 160 * thread0 in initarm so have to wait until it returns to enable it. 161 * If we were to enable it in initarm then any authentication when 162 * returning would fail as it was called with pointer authentication 163 * disabled. 164 */ 165 bl ptrauth_start 166 167 bl mi_startup 168 169 /* We should not get here */ 170 brk 0 171 172 .align 3 173.Lvirtdone: 174 .quad virtdone 175.Lbss: 176 .quad __bss_start 177.Lend: 178 .quad __bss_end 179END(_start) 180 181#ifdef SMP 182/* 183 * mpentry(unsigned long) 184 * 185 * Called by a core when it is being brought online. 186 * The data in x0 is passed straight to init_secondary. 187 */ 188ENTRY(mpentry) 189 /* Disable interrupts */ 190 msr daifset, #DAIF_INTR 191 192 /* Drop to EL1 */ 193 bl drop_to_el1 194 195 /* Set the context id */ 196 msr contextidr_el1, xzr 197 198 /* Load the kernel page table */ 199 adrp x24, pagetable_l0_ttbr1 200 add x24, x24, :lo12:pagetable_l0_ttbr1 201 /* Load the identity page table */ 202 adrp x27, pagetable_l0_ttbr0_boostrap 203 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 204 205 /* Enable the mmu */ 206 bl start_mmu 207 208 /* Load the new ttbr0 pagetable */ 209 adrp x27, pagetable_l0_ttbr0 210 add x27, x27, :lo12:pagetable_l0_ttbr0 211 212 /* Jump to the virtual address space */ 213 ldr x15, =mp_virtdone 214 br x15 215 216mp_virtdone: 217 /* Start using the AP boot stack */ 218 ldr x4, =bootstack 219 ldr x4, [x4] 220 mov sp, x4 221 222#if defined(PERTHREAD_SSP) 223 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 224 adrp x15, boot_canary 225 add x15, x15, :lo12:boot_canary 226 msr sp_el0, x15 227#endif 228 229 /* Load the kernel ttbr0 pagetable */ 230 msr ttbr0_el1, x27 231 isb 232 233 /* Invalidate the TLB */ 234 tlbi vmalle1 235 dsb sy 236 isb 237 238 b init_secondary 239END(mpentry) 240#endif 241 242/* 243 * If we are started in EL2, configure the required hypervisor 244 * registers and drop to EL1. 245 */ 246LENTRY(drop_to_el1) 247 mrs x23, CurrentEL 248 lsr x23, x23, #2 249 cmp x23, #0x2 250 b.eq 1f 251 ret 2521: 253 /* 254 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 255 * which may break address translation. 256 */ 257 dsb sy 258 mrs x2, sctlr_el2 259 bic x2, x2, SCTLR_M 260 msr sctlr_el2, x2 261 isb 262 263 /* Configure the Hypervisor */ 264 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 265 msr hcr_el2, x2 266 267 /* Stash value of HCR_EL2 for later */ 268 isb 269 mrs x4, hcr_el2 270 271 /* Load the Virtualization Process ID Register */ 272 mrs x2, midr_el1 273 msr vpidr_el2, x2 274 275 /* Load the Virtualization Multiprocess ID Register */ 276 mrs x2, mpidr_el1 277 msr vmpidr_el2, x2 278 279 /* Set the bits that need to be 1 in sctlr_el1 */ 280 ldr x2, .Lsctlr_res1 281 msr sctlr_el1, x2 282 283 /* 284 * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 285 * don't trap to EL2 for SIMD register usage to have at least a 286 * minimally usable system. 287 */ 288 tst x4, #HCR_E2H 289 mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 290 mov x5, #CPTR_FPEN /* HCR_E2H == 1 */ 291 csel x2, x3, x5, eq 292 msr cptr_el2, x2 293 294 /* Don't trap to EL2 for CP15 traps */ 295 msr hstr_el2, xzr 296 297 /* Enable access to the physical timers at EL1 */ 298 mrs x2, cnthctl_el2 299 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 300 msr cnthctl_el2, x2 301 302 /* Set the counter offset to a known value */ 303 msr cntvoff_el2, xzr 304 305 /* Hypervisor trap functions */ 306 adrp x2, hyp_stub_vectors 307 add x2, x2, :lo12:hyp_stub_vectors 308 msr vbar_el2, x2 309 310 /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 311 msr vttbr_el2, xzr 312 313 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 314 msr spsr_el2, x2 315 316 /* Configure GICv3 CPU interface */ 317 mrs x2, id_aa64pfr0_el1 318 /* Extract GIC bits from the register */ 319 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 320 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 321 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 322 b.ne 2f 323 324 mrs x2, icc_sre_el2 325 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 326 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 327 msr icc_sre_el2, x2 3282: 329 330 /* Set the address to return to our return address */ 331 msr elr_el2, x30 332 isb 333 334 eret 335 336 .align 3 337.Lsctlr_res1: 338 .quad SCTLR_RES1 339LEND(drop_to_el1) 340 341/* 342 * Get the delta between the physical address we were loaded to and the 343 * virtual address we expect to run from. This is used when building the 344 * initial page table. 345 */ 346LENTRY(get_virt_delta) 347 /* Load the physical address of virt_map */ 348 adrp x29, virt_map 349 add x29, x29, :lo12:virt_map 350 /* Load the virtual address of virt_map stored in virt_map */ 351 ldr x28, [x29] 352 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 353 sub x29, x29, x28 354 /* Find the load address for the kernel */ 355 mov x28, #(KERNBASE) 356 add x28, x28, x29 357 ret 358 359 .align 3 360virt_map: 361 .quad virt_map 362LEND(get_virt_delta) 363 364/* 365 * This builds the page tables containing the identity map, and the kernel 366 * virtual map. 367 * 368 * It relys on: 369 * We were loaded to an address that is on a 2MiB boundary 370 * All the memory must not cross a 1GiB boundaty 371 * x28 contains the physical address we were loaded from 372 * 373 * TODO: This is out of date. 374 * There are at least 5 pages before that address for the page tables 375 * The pages used are: 376 * - The Kernel L2 table 377 * - The Kernel L1 table 378 * - The Kernel L0 table (TTBR1) 379 * - The identity (PA = VA) L1 table 380 * - The identity (PA = VA) L0 table (TTBR0) 381 */ 382LENTRY(create_pagetables) 383 /* Save the Link register */ 384 mov x5, x30 385 386 /* Clean the page table */ 387 adrp x6, pagetable 388 add x6, x6, :lo12:pagetable 389 mov x26, x6 390 adrp x27, pagetable_end 391 add x27, x27, :lo12:pagetable_end 3921: 393 stp xzr, xzr, [x6], #16 394 stp xzr, xzr, [x6], #16 395 stp xzr, xzr, [x6], #16 396 stp xzr, xzr, [x6], #16 397 cmp x6, x27 398 b.lo 1b 399 400 /* 401 * Build the TTBR1 maps. 402 */ 403 404 /* Find the size of the kernel */ 405 mov x6, #(KERNBASE) 406 407#if defined(LINUX_BOOT_ABI) 408 /* X19 is used as 'map FDT data' flag */ 409 mov x19, xzr 410 411 /* No modules or FDT pointer ? */ 412 cbz x0, booti_no_fdt 413 414 /* 415 * Test if x0 points to modules descriptor(virtual address) or 416 * to FDT (physical address) 417 */ 418 cmp x0, x6 /* x6 is #(KERNBASE) */ 419 b.lo booti_fdt 420#endif 421 422 /* Booted with modules pointer */ 423 /* Find modulep - begin */ 424 sub x8, x0, x6 425 /* 426 * Add space for the module data. When PAGE_SIZE is 4k this will 427 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 428 * larger it will be at least as large as we use smaller level 3 429 * pages. 430 */ 431 ldr x7, =((6 * 1024 * 1024) - 1) 432 add x8, x8, x7 433 b common 434 435#if defined(LINUX_BOOT_ABI) 436booti_fdt: 437 /* Booted by U-Boot booti with FDT data */ 438 /* Set 'map FDT data' flag */ 439 mov x19, #1 440 441booti_no_fdt: 442 /* Booted by U-Boot booti without FTD data */ 443 /* Find the end - begin */ 444 ldr x7, .Lend 445 sub x8, x7, x6 446 447 /* 448 * Add one 2MiB page for copy of FDT data (maximum FDT size), 449 * one for metadata and round up 450 */ 451 ldr x7, =(3 * L2_SIZE - 1) 452 add x8, x8, x7 453#endif 454 455common: 456#if PAGE_SIZE != PAGE_SIZE_4K 457 /* 458 * Create L3 pages. The kernel will be loaded at a 2M aligned 459 * address, however L2 blocks are too large when the page size is 460 * not 4k to map the kernel with such an aligned address. However, 461 * when the page size is larger than 4k, L2 blocks are too large to 462 * map the kernel with such an alignment. 463 */ 464 465 /* Get the number of l3 pages to allocate, rounded down */ 466 lsr x10, x8, #(L3_SHIFT) 467 468 /* Create the kernel space L2 table */ 469 mov x6, x26 470 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 471 mov x8, #(KERNBASE) 472 mov x9, x28 473 bl build_l3_page_pagetable 474 475 /* Move to the l2 table */ 476 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 477 add x26, x26, x9 478 479 /* Link the l2 -> l3 table */ 480 mov x9, x6 481 mov x6, x26 482 bl link_l2_pagetable 483#else 484 /* Get the number of l2 pages to allocate, rounded down */ 485 lsr x10, x8, #(L2_SHIFT) 486 487 /* Create the kernel space L2 table */ 488 mov x6, x26 489 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 490 mov x8, #(KERNBASE) 491 mov x9, x28 492 bl build_l2_block_pagetable 493#endif 494 495 /* Move to the l1 table */ 496 add x26, x26, #PAGE_SIZE 497 498 /* Link the l1 -> l2 table */ 499 mov x9, x6 500 mov x6, x26 501 bl link_l1_pagetable 502 503 /* Move to the l0 table */ 504 add x24, x26, #PAGE_SIZE 505 506 /* Link the l0 -> l1 table */ 507 mov x9, x6 508 mov x6, x24 509 mov x10, #1 510 bl link_l0_pagetable 511 512 /* 513 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 514 * They are only needed early on, so the VA = PA map is uncached. 515 */ 516 add x27, x24, #PAGE_SIZE 517 518 mov x6, x27 /* The initial page table */ 519 520 /* Create the VA = PA map */ 521 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 522 adrp x16, _start 523 and x16, x16, #(~L2_OFFSET) 524 mov x9, x16 /* PA start */ 525 mov x8, x16 /* VA start (== PA start) */ 526 mov x10, #1 527 bl build_l2_block_pagetable 528 529#if defined(SOCDEV_PA) 530 /* Create a table for the UART */ 531 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 532 ldr x9, =(L2_SIZE) 533 add x16, x16, x9 /* VA start */ 534 mov x8, x16 535 536 /* Store the socdev virtual address */ 537 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 538 adrp x9, socdev_va 539 str x17, [x9, :lo12:socdev_va] 540 541 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 542 mov x10, #1 543 bl build_l2_block_pagetable 544#endif 545 546#if defined(LINUX_BOOT_ABI) 547 /* Map FDT data ? */ 548 cbz x19, 1f 549 550 /* Create the mapping for FDT data (2 MiB max) */ 551 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 552 ldr x9, =(L2_SIZE) 553 add x16, x16, x9 /* VA start */ 554 mov x8, x16 555 mov x9, x0 /* PA start */ 556 /* Update the module pointer to point at the allocated memory */ 557 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 558 add x0, x0, x8 /* Add the aligned virtual address */ 559 560 mov x10, #1 561 bl build_l2_block_pagetable 562 5631: 564#endif 565 566 /* Move to the l1 table */ 567 add x27, x27, #PAGE_SIZE 568 569 /* Link the l1 -> l2 table */ 570 mov x9, x6 571 mov x6, x27 572 bl link_l1_pagetable 573 574 /* Move to the l0 table */ 575 add x27, x27, #PAGE_SIZE 576 577 /* Link the l0 -> l1 table */ 578 mov x9, x6 579 mov x6, x27 580 mov x10, #1 581 bl link_l0_pagetable 582 583 /* Restore the Link register */ 584 mov x30, x5 585 ret 586LEND(create_pagetables) 587 588/* 589 * Builds an L0 -> L1 table descriptor 590 * 591 * x6 = L0 table 592 * x8 = Virtual Address 593 * x9 = L1 PA (trashed) 594 * x10 = Entry count (trashed) 595 * x11, x12 and x13 are trashed 596 */ 597LENTRY(link_l0_pagetable) 598 /* 599 * Link an L0 -> L1 table entry. 600 */ 601 /* Find the table index */ 602 lsr x11, x8, #L0_SHIFT 603 and x11, x11, #L0_ADDR_MASK 604 605 /* Build the L0 block entry */ 606 mov x12, #L0_TABLE 607 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 608 609 /* Only use the output address bits */ 610 lsr x9, x9, #PAGE_SHIFT 6111: orr x13, x12, x9, lsl #PAGE_SHIFT 612 613 /* Store the entry */ 614 str x13, [x6, x11, lsl #3] 615 616 sub x10, x10, #1 617 add x11, x11, #1 618 add x9, x9, #1 619 cbnz x10, 1b 620 621 ret 622LEND(link_l0_pagetable) 623 624/* 625 * Builds an L1 -> L2 table descriptor 626 * 627 * x6 = L1 table 628 * x8 = Virtual Address 629 * x9 = L2 PA (trashed) 630 * x11, x12 and x13 are trashed 631 */ 632LENTRY(link_l1_pagetable) 633 /* 634 * Link an L1 -> L2 table entry. 635 */ 636 /* Find the table index */ 637 lsr x11, x8, #L1_SHIFT 638 and x11, x11, #Ln_ADDR_MASK 639 640 /* Build the L1 block entry */ 641 mov x12, #L1_TABLE 642 643 /* Only use the output address bits */ 644 lsr x9, x9, #PAGE_SHIFT 645 orr x13, x12, x9, lsl #PAGE_SHIFT 646 647 /* Store the entry */ 648 str x13, [x6, x11, lsl #3] 649 650 ret 651LEND(link_l1_pagetable) 652 653/* 654 * Builds count 2 MiB page table entry 655 * x6 = L2 table 656 * x7 = Block attributes 657 * x8 = VA start 658 * x9 = PA start (trashed) 659 * x10 = Entry count (trashed) 660 * x11, x12 and x13 are trashed 661 */ 662LENTRY(build_l2_block_pagetable) 663 /* 664 * Build the L2 table entry. 665 */ 666 /* Find the table index */ 667 lsr x11, x8, #L2_SHIFT 668 and x11, x11, #Ln_ADDR_MASK 669 670 /* Build the L2 block entry */ 671 orr x12, x7, #L2_BLOCK 672 orr x12, x12, #(ATTR_DEFAULT) 673 orr x12, x12, #(ATTR_S1_UXN) 674 675 /* Only use the output address bits */ 676 lsr x9, x9, #L2_SHIFT 677 678 /* Set the physical address for this virtual address */ 6791: orr x13, x12, x9, lsl #L2_SHIFT 680 681 /* Store the entry */ 682 str x13, [x6, x11, lsl #3] 683 684 sub x10, x10, #1 685 add x11, x11, #1 686 add x9, x9, #1 687 cbnz x10, 1b 688 689 ret 690LEND(build_l2_block_pagetable) 691 692#if PAGE_SIZE != PAGE_SIZE_4K 693/* 694 * Builds an L2 -> L3 table descriptor 695 * 696 * x6 = L2 table 697 * x8 = Virtual Address 698 * x9 = L3 PA (trashed) 699 * x11, x12 and x13 are trashed 700 */ 701LENTRY(link_l2_pagetable) 702 /* 703 * Link an L2 -> L3 table entry. 704 */ 705 /* Find the table index */ 706 lsr x11, x8, #L2_SHIFT 707 and x11, x11, #Ln_ADDR_MASK 708 709 /* Build the L1 block entry */ 710 mov x12, #L2_TABLE 711 712 /* Only use the output address bits */ 713 lsr x9, x9, #PAGE_SHIFT 714 orr x13, x12, x9, lsl #PAGE_SHIFT 715 716 /* Store the entry */ 717 str x13, [x6, x11, lsl #3] 718 719 ret 720LEND(link_l2_pagetable) 721 722/* 723 * Builds count level 3 page table entries 724 * x6 = L3 table 725 * x7 = Block attributes 726 * x8 = VA start 727 * x9 = PA start (trashed) 728 * x10 = Entry count (trashed) 729 * x11, x12 and x13 are trashed 730 */ 731LENTRY(build_l3_page_pagetable) 732 /* 733 * Build the L3 table entry. 734 */ 735 /* Find the table index */ 736 lsr x11, x8, #L3_SHIFT 737 and x11, x11, #Ln_ADDR_MASK 738 739 /* Build the L3 page entry */ 740 orr x12, x7, #L3_PAGE 741 orr x12, x12, #(ATTR_DEFAULT) 742 orr x12, x12, #(ATTR_S1_UXN) 743 744 /* Only use the output address bits */ 745 lsr x9, x9, #L3_SHIFT 746 747 /* Set the physical address for this virtual address */ 7481: orr x13, x12, x9, lsl #L3_SHIFT 749 750 /* Store the entry */ 751 str x13, [x6, x11, lsl #3] 752 753 sub x10, x10, #1 754 add x11, x11, #1 755 add x9, x9, #1 756 cbnz x10, 1b 757 758 ret 759LEND(build_l3_page_pagetable) 760#endif 761 762LENTRY(start_mmu) 763 dsb sy 764 765 /* Load the exception vectors */ 766 ldr x2, =exception_vectors 767 msr vbar_el1, x2 768 769 /* Load ttbr0 and ttbr1 */ 770 msr ttbr0_el1, x27 771 msr ttbr1_el1, x24 772 isb 773 774 /* Clear the Monitor Debug System control register */ 775 msr mdscr_el1, xzr 776 777 /* Invalidate the TLB */ 778 tlbi vmalle1is 779 dsb ish 780 isb 781 782 ldr x2, mair 783 msr mair_el1, x2 784 785 /* 786 * Setup TCR according to the PARange and ASIDBits fields 787 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 788 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 789 * to 1 only if the ASIDBits field equals 0b0010. 790 */ 791 ldr x2, tcr 792 mrs x3, id_aa64mmfr0_el1 793 794 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 795 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 796 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 797 798 /* Check if the HW supports 16 bit ASIDS */ 799 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 800 /* If so x3 == 1, else x3 == 0 */ 801 cset x3, eq 802 /* Set TCR.AS with x3 */ 803 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 804 805 /* 806 * Check if the HW supports access flag and dirty state updates, 807 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 808 */ 809 mrs x3, id_aa64mmfr1_el1 810 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 811 cmp x3, #1 812 b.ne 1f 813 orr x2, x2, #(TCR_HA) 814 b 2f 8151: 816 cmp x3, #2 817 b.ne 2f 818 orr x2, x2, #(TCR_HA | TCR_HD) 8192: 820 msr tcr_el1, x2 821 822 /* 823 * Setup SCTLR. 824 */ 825 ldr x2, sctlr_set 826 ldr x3, sctlr_clear 827 mrs x1, sctlr_el1 828 bic x1, x1, x3 /* Clear the required bits */ 829 orr x1, x1, x2 /* Set the required bits */ 830 msr sctlr_el1, x1 831 isb 832 833 ret 834 835 .align 3 836mair: 837 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 838 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 839 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 840 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 841 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 842tcr: 843#if PAGE_SIZE == PAGE_SIZE_4K 844#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 845#elif PAGE_SIZE == PAGE_SIZE_16K 846#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 847#else 848#error Unsupported page size 849#endif 850 851 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 852 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 853sctlr_set: 854 /* Bits to set */ 855 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 856 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 857 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 858 SCTLR_M | SCTLR_CP15BEN) 859sctlr_clear: 860 /* Bits to clear */ 861 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 862 SCTLR_ITD | SCTLR_A) 863LEND(start_mmu) 864 865ENTRY(abort) 866 b abort 867END(abort) 868 869 .section .init_pagetable, "aw", %nobits 870 .align PAGE_SHIFT 871 /* 872 * 6 initial tables (in the following order): 873 * L2 for kernel (High addresses) 874 * L1 for kernel 875 * L0 for kernel 876 * L1 bootstrap for user (Low addresses) 877 * L0 bootstrap for user 878 * L0 for user 879 */ 880 .globl pagetable_l0_ttbr1 881pagetable: 882#if PAGE_SIZE != PAGE_SIZE_4K 883 .space (PAGE_SIZE * L3_PAGE_COUNT) 884pagetable_l2_ttbr1: 885#endif 886 .space PAGE_SIZE 887pagetable_l1_ttbr1: 888 .space PAGE_SIZE 889pagetable_l0_ttbr1: 890 .space PAGE_SIZE 891pagetable_l2_ttbr0_bootstrap: 892 .space PAGE_SIZE 893pagetable_l1_ttbr0_bootstrap: 894 .space PAGE_SIZE 895pagetable_l0_ttbr0_boostrap: 896 .space PAGE_SIZE 897pagetable_l0_ttbr0: 898 .space PAGE_SIZE 899 900pagetable_end: 901 902el2_pagetable: 903 .space PAGE_SIZE 904 905 .align 4 906initstack: 907 .space (PAGE_SIZE * KSTACK_PAGES) 908initstack_end: 909 910 911.text 912EENTRY(aarch32_sigcode) 913 .word 0xe1a0000d // mov r0, sp 914 .word 0xe2800040 // add r0, r0, #SIGF_UC 915 .word 0xe59f700c // ldr r7, [pc, #12] 916 .word 0xef000000 // swi #0 917 .word 0xe59f7008 // ldr r7, [pc, #8] 918 .word 0xef000000 // swi #0 919 .word 0xeafffffa // b . - 16 920EEND(aarch32_sigcode) 921 .word SYS_sigreturn 922 .word SYS_exit 923 .align 3 924aarch32_esigcode: 925 .data 926 .global sz_aarch32_sigcode 927sz_aarch32_sigcode: 928 .quad aarch32_esigcode - aarch32_sigcode 929