1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) 42 43 .globl kernbase 44 .set kernbase, KERNBASE 45 46 47/* U-Boot booti related constants. */ 48#if defined(LINUX_BOOT_ABI) 49#ifndef UBOOT_IMAGE_OFFSET 50#define UBOOT_IMAGE_OFFSET 0 /* Image offset from start of */ 51#endif /* 2 MiB page */ 52 53#ifndef UBOOT_IMAGE_SIZE /* Total size of image */ 54#define UBOOT_IMAGE_SIZE _end - _start 55#endif 56 57#ifndef UBOOT_IMAGE_FLAGS 58#define UBOOT_IMAGE_FLAGS 0 /* LE kernel, unspecified */ 59#endif /* page size */ 60#endif /* defined(LINUX_BOOT_ABI) */ 61 62/* 63 * We assume: 64 * MMU on with an identity map, or off 65 * D-Cache: off 66 * I-Cache: on or off 67 * We are loaded at a 2MiB aligned address 68 */ 69 70ENTRY(_start) 71#if defined(LINUX_BOOT_ABI) 72 /* U-boot image header */ 73 b 1f /* code 0 */ 74 .long 0 /* code 1 */ 75 .quad UBOOT_IMAGE_OFFSET /* Image offset in 2 MiB page, LE */ 76 .quad UBOOT_IMAGE_SIZE /* Image size, LE */ 77 .quad UBOOT_IMAGE_FLAGS /* Flags for kernel. LE */ 78 .quad 0 /* Reserved */ 79 .quad 0 /* Reserved */ 80 .quad 0 /* Reserved */ 81 .long 0x644d5241 /* Magic "ARM\x64", LE */ 82 .long 0 /* Reserved for PE COFF offset*/ 831: 84#endif /* defined(LINUX_BOOT_ABI) */ 85 86 /* Drop to EL1 */ 87 bl drop_to_el1 88 89 /* 90 * Disable the MMU. We may have entered the kernel with it on and 91 * will need to update the tables later. If this has been set up 92 * with anything other than a VA == PA map then this will fail, 93 * but in this case the code to find where we are running from 94 * would have also failed. 95 */ 96 dsb sy 97 mrs x2, sctlr_el1 98 bic x2, x2, SCTLR_M 99 msr sctlr_el1, x2 100 isb 101 102 /* Set the context id */ 103 msr contextidr_el1, xzr 104 105 /* Get the virt -> phys offset */ 106 bl get_virt_delta 107 108 /* 109 * At this point: 110 * x29 = PA - VA 111 * x28 = Our physical load address 112 */ 113 114 /* Create the page tables */ 115 bl create_pagetables 116 117 /* 118 * At this point: 119 * x27 = TTBR0 table 120 * x26 = Kernel L1 table 121 * x24 = TTBR1 table 122 */ 123 124 /* Enable the mmu */ 125 bl start_mmu 126 127 /* Load the new ttbr0 pagetable */ 128 adrp x27, pagetable_l0_ttbr0 129 add x27, x27, :lo12:pagetable_l0_ttbr0 130 131 /* Jump to the virtual address space */ 132 ldr x15, .Lvirtdone 133 br x15 134 135virtdone: 136 /* Set up the stack */ 137 adrp x25, initstack_end 138 add x25, x25, :lo12:initstack_end 139 mov sp, x25 140 sub sp, sp, #PCB_SIZE 141 142 /* Zero the BSS */ 143 ldr x15, .Lbss 144 ldr x14, .Lend 1451: 146 str xzr, [x15], #8 147 cmp x15, x14 148 b.lo 1b 149 150 /* Backup the module pointer */ 151 mov x1, x0 152 153 /* Make the page table base a virtual address */ 154 sub x26, x26, x29 155 sub x24, x24, x29 156 157 sub sp, sp, #BOOTPARAMS_SIZE 158 mov x0, sp 159 160 /* Degate the delda so it is VA -> PA */ 161 neg x29, x29 162 163 str x1, [x0, #BP_MODULEP] 164 str x26, [x0, #BP_KERN_L1PT] 165 str x29, [x0, #BP_KERN_DELTA] 166 adrp x25, initstack 167 add x25, x25, :lo12:initstack 168 str x25, [x0, #BP_KERN_STACK] 169 str x24, [x0, #BP_KERN_L0PT] 170 str x27, [x0, #BP_KERN_TTBR0] 171 str x23, [x0, #BP_BOOT_EL] 172 173 /* trace back starts here */ 174 mov fp, #0 175 /* Branch to C code */ 176 bl initarm 177 /* We are done with the boot params */ 178 add sp, sp, #BOOTPARAMS_SIZE 179 bl mi_startup 180 181 /* We should not get here */ 182 brk 0 183 184 .align 3 185.Lvirtdone: 186 .quad virtdone 187.Lbss: 188 .quad __bss_start 189.Lend: 190 .quad __bss_end 191END(_start) 192 193#ifdef SMP 194/* 195 * mpentry(unsigned long) 196 * 197 * Called by a core when it is being brought online. 198 * The data in x0 is passed straight to init_secondary. 199 */ 200ENTRY(mpentry) 201 /* Disable interrupts */ 202 msr daifset, #2 203 204 /* Drop to EL1 */ 205 bl drop_to_el1 206 207 /* Set the context id */ 208 msr contextidr_el1, xzr 209 210 /* Load the kernel page table */ 211 adrp x24, pagetable_l0_ttbr1 212 add x24, x24, :lo12:pagetable_l0_ttbr1 213 /* Load the identity page table */ 214 adrp x27, pagetable_l0_ttbr0_boostrap 215 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 216 217 /* Enable the mmu */ 218 bl start_mmu 219 220 /* Load the new ttbr0 pagetable */ 221 adrp x27, pagetable_l0_ttbr0 222 add x27, x27, :lo12:pagetable_l0_ttbr0 223 224 /* Jump to the virtual address space */ 225 ldr x15, =mp_virtdone 226 br x15 227 228mp_virtdone: 229 /* Start using the AP boot stack */ 230 ldr x4, =bootstack 231 ldr x4, [x4] 232 mov sp, x4 233 234 /* Load the kernel ttbr0 pagetable */ 235 msr ttbr0_el1, x27 236 isb 237 238 /* Invalidate the TLB */ 239 tlbi vmalle1 240 dsb sy 241 isb 242 243 b init_secondary 244END(mpentry) 245#endif 246 247/* 248 * If we are started in EL2, configure the required hypervisor 249 * registers and drop to EL1. 250 */ 251LENTRY(drop_to_el1) 252 mrs x23, CurrentEL 253 lsr x23, x23, #2 254 cmp x23, #0x2 255 b.eq 1f 256 ret 2571: 258 /* Configure the Hypervisor */ 259 mov x2, #(HCR_RW) 260 msr hcr_el2, x2 261 262 /* Load the Virtualization Process ID Register */ 263 mrs x2, midr_el1 264 msr vpidr_el2, x2 265 266 /* Load the Virtualization Multiprocess ID Register */ 267 mrs x2, mpidr_el1 268 msr vmpidr_el2, x2 269 270 /* Set the bits that need to be 1 in sctlr_el1 */ 271 ldr x2, .Lsctlr_res1 272 msr sctlr_el1, x2 273 274 /* Don't trap to EL2 for exceptions */ 275 mov x2, #CPTR_RES1 276 msr cptr_el2, x2 277 278 /* Don't trap to EL2 for CP15 traps */ 279 msr hstr_el2, xzr 280 281 /* Enable access to the physical timers at EL1 */ 282 mrs x2, cnthctl_el2 283 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 284 msr cnthctl_el2, x2 285 286 /* Set the counter offset to a known value */ 287 msr cntvoff_el2, xzr 288 289 /* Hypervisor trap functions */ 290 adrp x2, hyp_vectors 291 add x2, x2, :lo12:hyp_vectors 292 msr vbar_el2, x2 293 294 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 295 msr spsr_el2, x2 296 297 /* Configure GICv3 CPU interface */ 298 mrs x2, id_aa64pfr0_el1 299 /* Extract GIC bits from the register */ 300 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 301 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 302 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 303 b.ne 2f 304 305 mrs x2, icc_sre_el2 306 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 307 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 308 msr icc_sre_el2, x2 3092: 310 311 /* Set the address to return to our return address */ 312 msr elr_el2, x30 313 isb 314 315 eret 316 317 .align 3 318.Lsctlr_res1: 319 .quad SCTLR_RES1 320LEND(drop_to_el1) 321 322#define VECT_EMPTY \ 323 .align 7; \ 324 1: b 1b 325 326 .align 11 327hyp_vectors: 328 VECT_EMPTY /* Synchronous EL2t */ 329 VECT_EMPTY /* IRQ EL2t */ 330 VECT_EMPTY /* FIQ EL2t */ 331 VECT_EMPTY /* Error EL2t */ 332 333 VECT_EMPTY /* Synchronous EL2h */ 334 VECT_EMPTY /* IRQ EL2h */ 335 VECT_EMPTY /* FIQ EL2h */ 336 VECT_EMPTY /* Error EL2h */ 337 338 VECT_EMPTY /* Synchronous 64-bit EL1 */ 339 VECT_EMPTY /* IRQ 64-bit EL1 */ 340 VECT_EMPTY /* FIQ 64-bit EL1 */ 341 VECT_EMPTY /* Error 64-bit EL1 */ 342 343 VECT_EMPTY /* Synchronous 32-bit EL1 */ 344 VECT_EMPTY /* IRQ 32-bit EL1 */ 345 VECT_EMPTY /* FIQ 32-bit EL1 */ 346 VECT_EMPTY /* Error 32-bit EL1 */ 347 348/* 349 * Get the delta between the physical address we were loaded to and the 350 * virtual address we expect to run from. This is used when building the 351 * initial page table. 352 */ 353LENTRY(get_virt_delta) 354 /* Load the physical address of virt_map */ 355 adrp x29, virt_map 356 add x29, x29, :lo12:virt_map 357 /* Load the virtual address of virt_map stored in virt_map */ 358 ldr x28, [x29] 359 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 360 sub x29, x29, x28 361 /* Find the load address for the kernel */ 362 mov x28, #(KERNBASE) 363 add x28, x28, x29 364 ret 365 366 .align 3 367virt_map: 368 .quad virt_map 369LEND(get_virt_delta) 370 371/* 372 * This builds the page tables containing the identity map, and the kernel 373 * virtual map. 374 * 375 * It relys on: 376 * We were loaded to an address that is on a 2MiB boundary 377 * All the memory must not cross a 1GiB boundaty 378 * x28 contains the physical address we were loaded from 379 * 380 * TODO: This is out of date. 381 * There are at least 5 pages before that address for the page tables 382 * The pages used are: 383 * - The Kernel L2 table 384 * - The Kernel L1 table 385 * - The Kernel L0 table (TTBR1) 386 * - The identity (PA = VA) L1 table 387 * - The identity (PA = VA) L0 table (TTBR0) 388 * - The DMAP L1 tables 389 */ 390LENTRY(create_pagetables) 391 /* Save the Link register */ 392 mov x5, x30 393 394 /* Clean the page table */ 395 adrp x6, pagetable 396 add x6, x6, :lo12:pagetable 397 mov x26, x6 398 adrp x27, pagetable_end 399 add x27, x27, :lo12:pagetable_end 4001: 401 stp xzr, xzr, [x6], #16 402 stp xzr, xzr, [x6], #16 403 stp xzr, xzr, [x6], #16 404 stp xzr, xzr, [x6], #16 405 cmp x6, x27 406 b.lo 1b 407 408 /* 409 * Build the TTBR1 maps. 410 */ 411 412 /* Find the size of the kernel */ 413 mov x6, #(KERNBASE) 414 415#if defined(LINUX_BOOT_ABI) 416 /* X19 is used as 'map FDT data' flag */ 417 mov x19, xzr 418 419 /* No modules or FDT pointer ? */ 420 cbz x0, booti_no_fdt 421 422 /* 423 * Test if x0 points to modules descriptor(virtual address) or 424 * to FDT (physical address) 425 */ 426 cmp x0, x6 /* x6 is #(KERNBASE) */ 427 b.lo booti_fdt 428#endif 429 430 /* Booted with modules pointer */ 431 /* Find modulep - begin */ 432 sub x8, x0, x6 433 /* Add two 2MiB pages for the module data and round up */ 434 ldr x7, =(3 * L2_SIZE - 1) 435 add x8, x8, x7 436 b common 437 438#if defined(LINUX_BOOT_ABI) 439booti_fdt: 440 /* Booted by U-Boot booti with FDT data */ 441 /* Set 'map FDT data' flag */ 442 mov x19, #1 443 444booti_no_fdt: 445 /* Booted by U-Boot booti without FTD data */ 446 /* Find the end - begin */ 447 ldr x7, .Lend 448 sub x8, x7, x6 449 450 /* 451 * Add one 2MiB page for copy of FDT data (maximum FDT size), 452 * one for metadata and round up 453 */ 454 ldr x7, =(3 * L2_SIZE - 1) 455 add x8, x8, x7 456#endif 457 458common: 459 /* Get the number of l2 pages to allocate, rounded down */ 460 lsr x10, x8, #(L2_SHIFT) 461 462 /* Create the kernel space L2 table */ 463 mov x6, x26 464 mov x7, #VM_MEMATTR_WRITE_BACK 465 mov x8, #(KERNBASE & L2_BLOCK_MASK) 466 mov x9, x28 467 bl build_l2_block_pagetable 468 469 /* Move to the l1 table */ 470 add x26, x26, #PAGE_SIZE 471 472 /* Link the l1 -> l2 table */ 473 mov x9, x6 474 mov x6, x26 475 bl link_l1_pagetable 476 477 /* Move to the l0 table */ 478 add x24, x26, #PAGE_SIZE 479 480 /* Link the l0 -> l1 table */ 481 mov x9, x6 482 mov x6, x24 483 mov x10, #1 484 bl link_l0_pagetable 485 486 /* Link the DMAP tables */ 487 ldr x8, =DMAP_MIN_ADDRESS 488 adrp x9, pagetable_dmap 489 add x9, x9, :lo12:pagetable_dmap 490 mov x10, #DMAP_TABLES 491 bl link_l0_pagetable 492 493 /* 494 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 495 * They are only needed early on, so the VA = PA map is uncached. 496 */ 497 add x27, x24, #PAGE_SIZE 498 499 mov x6, x27 /* The initial page table */ 500#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 501 /* Create a table for the UART */ 502 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 503 mov x8, #(SOCDEV_VA) /* VA start */ 504 mov x9, #(SOCDEV_PA) /* PA start */ 505 mov x10, #1 506 bl build_l1_block_pagetable 507#endif 508 509#if defined(LINUX_BOOT_ABI) 510 /* Map FDT data ? */ 511 cbz x19, 1f 512 513 /* Create the identity mapping for FDT data (2 MiB max) */ 514 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 515 mov x9, x0 516 mov x8, x0 /* VA start (== PA start) */ 517 mov x10, #1 518 bl build_l1_block_pagetable 519 5201: 521#endif 522 523 /* Create the VA = PA map */ 524 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 525 mov x9, x28 526 mov x8, x9 /* VA start (== PA start) */ 527 mov x10, #1 528 bl build_l1_block_pagetable 529 530 /* Move to the l0 table */ 531 add x27, x27, #PAGE_SIZE 532 533 /* Link the l0 -> l1 table */ 534 mov x9, x6 535 mov x6, x27 536 mov x10, #1 537 bl link_l0_pagetable 538 539 /* Restore the Link register */ 540 mov x30, x5 541 ret 542LEND(create_pagetables) 543 544/* 545 * Builds an L0 -> L1 table descriptor 546 * 547 * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 548 * within it by build_l1_block_pagetable. 549 * 550 * x6 = L0 table 551 * x8 = Virtual Address 552 * x9 = L1 PA (trashed) 553 * x10 = Entry count 554 * x11, x12 and x13 are trashed 555 */ 556LENTRY(link_l0_pagetable) 557 /* 558 * Link an L0 -> L1 table entry. 559 */ 560 /* Find the table index */ 561 lsr x11, x8, #L0_SHIFT 562 and x11, x11, #L0_ADDR_MASK 563 564 /* Build the L0 block entry */ 565 mov x12, #L0_TABLE 566 567 /* Only use the output address bits */ 568 lsr x9, x9, #PAGE_SHIFT 5691: orr x13, x12, x9, lsl #PAGE_SHIFT 570 571 /* Store the entry */ 572 str x13, [x6, x11, lsl #3] 573 574 sub x10, x10, #1 575 add x11, x11, #1 576 add x9, x9, #1 577 cbnz x10, 1b 578 579 ret 580LEND(link_l0_pagetable) 581 582/* 583 * Builds an L1 -> L2 table descriptor 584 * 585 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 586 * within it by build_l2_block_pagetable. 587 * 588 * x6 = L1 table 589 * x8 = Virtual Address 590 * x9 = L2 PA (trashed) 591 * x11, x12 and x13 are trashed 592 */ 593LENTRY(link_l1_pagetable) 594 /* 595 * Link an L1 -> L2 table entry. 596 */ 597 /* Find the table index */ 598 lsr x11, x8, #L1_SHIFT 599 and x11, x11, #Ln_ADDR_MASK 600 601 /* Build the L1 block entry */ 602 mov x12, #L1_TABLE 603 604 /* Only use the output address bits */ 605 lsr x9, x9, #PAGE_SHIFT 606 orr x13, x12, x9, lsl #PAGE_SHIFT 607 608 /* Store the entry */ 609 str x13, [x6, x11, lsl #3] 610 611 ret 612LEND(link_l1_pagetable) 613 614/* 615 * Builds count 1 GiB page table entry 616 * x6 = L1 table 617 * x7 = Variable lower block attributes 618 * x8 = VA start 619 * x9 = PA start (trashed) 620 * x10 = Entry count 621 * x11, x12 and x13 are trashed 622 */ 623LENTRY(build_l1_block_pagetable) 624 /* 625 * Build the L1 table entry. 626 */ 627 /* Find the table index */ 628 lsr x11, x8, #L1_SHIFT 629 and x11, x11, #Ln_ADDR_MASK 630 631 /* Build the L1 block entry */ 632 orr x12, x7, #L1_BLOCK 633 orr x12, x12, #(ATTR_DEFAULT) 634 635 /* Only use the output address bits */ 636 lsr x9, x9, #L1_SHIFT 637 638 /* Set the physical address for this virtual address */ 6391: orr x13, x12, x9, lsl #L1_SHIFT 640 641 /* Store the entry */ 642 str x13, [x6, x11, lsl #3] 643 644 sub x10, x10, #1 645 add x11, x11, #1 646 add x9, x9, #1 647 cbnz x10, 1b 648 649 ret 650LEND(build_l1_block_pagetable) 651 652/* 653 * Builds count 2 MiB page table entry 654 * x6 = L2 table 655 * x7 = Type (0 = Device, 1 = Normal) 656 * x8 = VA start 657 * x9 = PA start (trashed) 658 * x10 = Entry count 659 * x11, x12 and x13 are trashed 660 */ 661LENTRY(build_l2_block_pagetable) 662 /* 663 * Build the L2 table entry. 664 */ 665 /* Find the table index */ 666 lsr x11, x8, #L2_SHIFT 667 and x11, x11, #Ln_ADDR_MASK 668 669 /* Build the L2 block entry */ 670 lsl x12, x7, #2 671 orr x12, x12, #L2_BLOCK 672 orr x12, x12, #(ATTR_DEFAULT) 673 orr x12, x12, #(ATTR_S1_UXN) 674 675 /* Only use the output address bits */ 676 lsr x9, x9, #L2_SHIFT 677 678 /* Set the physical address for this virtual address */ 6791: orr x13, x12, x9, lsl #L2_SHIFT 680 681 /* Store the entry */ 682 str x13, [x6, x11, lsl #3] 683 684 sub x10, x10, #1 685 add x11, x11, #1 686 add x9, x9, #1 687 cbnz x10, 1b 688 689 ret 690LEND(build_l2_block_pagetable) 691 692LENTRY(start_mmu) 693 dsb sy 694 695 /* Load the exception vectors */ 696 ldr x2, =exception_vectors 697 msr vbar_el1, x2 698 699 /* Load ttbr0 and ttbr1 */ 700 msr ttbr0_el1, x27 701 msr ttbr1_el1, x24 702 isb 703 704 /* Clear the Monitor Debug System control register */ 705 msr mdscr_el1, xzr 706 707 /* Invalidate the TLB */ 708 tlbi vmalle1is 709 dsb ish 710 isb 711 712 ldr x2, mair 713 msr mair_el1, x2 714 715 /* 716 * Setup TCR according to the PARange and ASIDBits fields 717 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 718 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 719 * to 1 only if the ASIDBits field equals 0b0010. 720 */ 721 ldr x2, tcr 722 mrs x3, id_aa64mmfr0_el1 723 724 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 725 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 726 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 727 728 /* Check if the HW supports 16 bit ASIDS */ 729 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 730 /* If so x3 == 1, else x3 == 0 */ 731 cset x3, eq 732 /* Set TCR.AS with x3 */ 733 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 734 735 /* 736 * Check if the HW supports access flag and dirty state updates, 737 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 738 */ 739 mrs x3, id_aa64mmfr1_el1 740 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 741 cmp x3, #1 742 b.ne 1f 743 orr x2, x2, #(TCR_HA) 744 b 2f 7451: 746 cmp x3, #2 747 b.ne 2f 748 orr x2, x2, #(TCR_HA | TCR_HD) 7492: 750 msr tcr_el1, x2 751 752 /* 753 * Setup SCTLR. 754 */ 755 ldr x2, sctlr_set 756 ldr x3, sctlr_clear 757 mrs x1, sctlr_el1 758 bic x1, x1, x3 /* Clear the required bits */ 759 orr x1, x1, x2 /* Set the required bits */ 760 msr sctlr_el1, x1 761 isb 762 763 ret 764 765 .align 3 766mair: 767 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | \ 768 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 769 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 770 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) 771tcr: 772 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG1_4K | \ 773 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 774sctlr_set: 775 /* Bits to set */ 776 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 777 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 778 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 779 SCTLR_M | SCTLR_CP15BEN) 780sctlr_clear: 781 /* Bits to clear */ 782 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 783 SCTLR_ITD | SCTLR_A) 784LEND(start_mmu) 785 786ENTRY(abort) 787 b abort 788END(abort) 789 790 .align 3 791init_pt_va: 792 .quad pagetable /* XXX: Keep page tables VA */ 793 794 .section .init_pagetable, "aw", %nobits 795 .align PAGE_SHIFT 796 /* 797 * 6 initial tables (in the following order): 798 * L2 for kernel (High addresses) 799 * L1 for kernel 800 * L0 for kernel 801 * L1 bootstrap for user (Low addresses) 802 * L0 bootstrap for user 803 * L0 for user 804 */ 805pagetable: 806 .space PAGE_SIZE 807pagetable_l1_ttbr1: 808 .space PAGE_SIZE 809pagetable_l0_ttbr1: 810 .space PAGE_SIZE 811pagetable_l1_ttbr0_bootstrap: 812 .space PAGE_SIZE 813pagetable_l0_ttbr0_boostrap: 814 .space PAGE_SIZE 815pagetable_l0_ttbr0: 816 .space PAGE_SIZE 817 818 .globl pagetable_dmap 819pagetable_dmap: 820 .space PAGE_SIZE * DMAP_TABLES 821pagetable_end: 822 823el2_pagetable: 824 .space PAGE_SIZE 825 826 .globl init_pt_va 827 828 .align 4 829initstack: 830 .space (PAGE_SIZE * KSTACK_PAGES) 831initstack_end: 832 833 834ENTRY(sigcode) 835 mov x0, sp 836 add x0, x0, #SF_UC 837 8381: 839 mov x8, #SYS_sigreturn 840 svc 0 841 842 /* sigreturn failed, exit */ 843 mov x8, #SYS_exit 844 svc 0 845 846 b 1b 847END(sigcode) 848 /* This may be copied to the stack, keep it 16-byte aligned */ 849 .align 3 850esigcode: 851 852 .data 853 .align 3 854 .global szsigcode 855szsigcode: 856 .quad esigcode - sigcode 857 858ENTRY(aarch32_sigcode) 859 .word 0xe1a0000d // mov r0, sp 860 .word 0xe2800040 // add r0, r0, #SIGF_UC 861 .word 0xe59f700c // ldr r7, [pc, #12] 862 .word 0xef000000 // swi #0 863 .word 0xe59f7008 // ldr r7, [pc, #8] 864 .word 0xef000000 // swi #0 865 .word 0xeafffffa // b . - 16 866END(aarch32_sigcode) 867 .word SYS_sigreturn 868 .word SYS_exit 869 .align 3 870aarch32_esigcode: 871 .data 872 .global sz_aarch32_sigcode 873sz_aarch32_sigcode: 874 .quad aarch32_esigcode - aarch32_sigcode 875