1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) 42 43 .globl kernbase 44 .set kernbase, KERNBASE 45 46 47/* U-Boot booti related constants. */ 48#if defined(LINUX_BOOT_ABI) 49#define FDT_MAGIC 0xEDFE0DD0 /* FDT blob Magic */ 50 51#ifndef UBOOT_IMAGE_OFFSET 52#define UBOOT_IMAGE_OFFSET 0 /* Image offset from start of */ 53#endif /* 2 MiB page */ 54 55#ifndef UBOOT_IMAGE_SIZE /* Total size of image */ 56#define UBOOT_IMAGE_SIZE _end - _start 57#endif 58 59#ifndef UBOOT_IMAGE_FLAGS 60#define UBOOT_IMAGE_FLAGS 0 /* LE kernel, unspecified */ 61#endif /* page size */ 62#endif /* defined(LINUX_BOOT_ABI) */ 63 64/* 65 * We assume: 66 * MMU on with an identity map, or off 67 * D-Cache: off 68 * I-Cache: on or off 69 * We are loaded at a 2MiB aligned address 70 */ 71 72 .text 73 .globl _start 74_start: 75#if defined(LINUX_BOOT_ABI) 76 /* U-boot image header */ 77 b 1f /* code 0 */ 78 .long 0 /* code 1 */ 79 .quad UBOOT_IMAGE_OFFSET /* Image offset in 2 MiB page, LE */ 80 .quad UBOOT_IMAGE_SIZE /* Image size, LE */ 81 .quad UBOOT_IMAGE_FLAGS /* Flags for kernel. LE */ 82 .quad 0 /* Reserved */ 83 .quad 0 /* Reserved */ 84 .quad 0 /* Reserved */ 85 .long 0x644d5241 /* Magic "ARM\x64", LE */ 86 .long 0 /* Reserved for PE COFF offset*/ 871: 88#endif /* defined(LINUX_BOOT_ABI) */ 89 90 /* Drop to EL1 */ 91 bl drop_to_el1 92 93 /* 94 * Disable the MMU. We may have entered the kernel with it on and 95 * will need to update the tables later. If this has been set up 96 * with anything other than a VA == PA map then this will fail, 97 * but in this case the code to find where we are running from 98 * would have also failed. 99 */ 100 dsb sy 101 mrs x2, sctlr_el1 102 bic x2, x2, SCTLR_M 103 msr sctlr_el1, x2 104 isb 105 106 /* Set the context id */ 107 msr contextidr_el1, xzr 108 109 /* Get the virt -> phys offset */ 110 bl get_virt_delta 111 112 /* 113 * At this point: 114 * x29 = PA - VA 115 * x28 = Our physical load address 116 */ 117 118 /* Create the page tables */ 119 bl create_pagetables 120 121 /* 122 * At this point: 123 * x27 = TTBR0 table 124 * x26 = Kernel L1 table 125 * x24 = TTBR1 table 126 */ 127 128 /* Enable the mmu */ 129 bl start_mmu 130 131 /* Load the new ttbr0 pagetable */ 132 adr x27, pagetable_l0_ttbr0 133 134 /* Jump to the virtual address space */ 135 ldr x15, .Lvirtdone 136 br x15 137 138virtdone: 139 /* Set up the stack */ 140 adr x25, initstack_end 141 mov sp, x25 142 sub sp, sp, #PCB_SIZE 143 144 /* Zero the BSS */ 145 ldr x15, .Lbss 146 ldr x14, .Lend 1471: 148 str xzr, [x15], #8 149 cmp x15, x14 150 b.lo 1b 151 152 /* Backup the module pointer */ 153 mov x1, x0 154 155 /* Make the page table base a virtual address */ 156 sub x26, x26, x29 157 sub x24, x24, x29 158 159 sub sp, sp, #BOOTPARAMS_SIZE 160 mov x0, sp 161 162 /* Degate the delda so it is VA -> PA */ 163 neg x29, x29 164 165 str x1, [x0, #BP_MODULEP] 166 str x26, [x0, #BP_KERN_L1PT] 167 str x29, [x0, #BP_KERN_DELTA] 168 adr x25, initstack 169 str x25, [x0, #BP_KERN_STACK] 170 str x24, [x0, #BP_KERN_L0PT] 171 str x23, [x0, #BP_BOOT_EL] 172 str x27, [x0, 40] /* kern_ttbr0 */ 173 174 /* trace back starts here */ 175 mov fp, #0 176 /* Branch to C code */ 177 bl initarm 178 bl mi_startup 179 180 /* We should not get here */ 181 brk 0 182 183 .align 3 184.Lvirtdone: 185 .quad virtdone 186.Lbss: 187 .quad __bss_start 188.Lend: 189 .quad _end 190 191#ifdef SMP 192/* 193 * mpentry(unsigned long) 194 * 195 * Called by a core when it is being brought online. 196 * The data in x0 is passed straight to init_secondary. 197 */ 198ENTRY(mpentry) 199 /* Disable interrupts */ 200 msr daifset, #2 201 202 /* Drop to EL1 */ 203 bl drop_to_el1 204 205 /* Set the context id */ 206 msr contextidr_el1, xzr 207 208 /* Load the kernel page table */ 209 adr x24, pagetable_l0_ttbr1 210 /* Load the identity page table */ 211 adr x27, pagetable_l0_ttbr0_boostrap 212 213 /* Enable the mmu */ 214 bl start_mmu 215 216 /* Load the new ttbr0 pagetable */ 217 adr x27, pagetable_l0_ttbr0 218 219 /* Jump to the virtual address space */ 220 ldr x15, =mp_virtdone 221 br x15 222 223mp_virtdone: 224 /* Start using the AP boot stack */ 225 ldr x4, =bootstack 226 ldr x4, [x4] 227 mov sp, x4 228 229 /* Load the kernel ttbr0 pagetable */ 230 msr ttbr0_el1, x27 231 isb 232 233 /* Invalidate the TLB */ 234 tlbi vmalle1 235 dsb sy 236 isb 237 238 b init_secondary 239END(mpentry) 240#endif 241 242/* 243 * If we are started in EL2, configure the required hypervisor 244 * registers and drop to EL1. 245 */ 246drop_to_el1: 247 mrs x23, CurrentEL 248 lsr x23, x23, #2 249 cmp x23, #0x2 250 b.eq 1f 251 ret 2521: 253 /* Configure the Hypervisor */ 254 mov x2, #(HCR_RW) 255 msr hcr_el2, x2 256 257 /* Load the Virtualization Process ID Register */ 258 mrs x2, midr_el1 259 msr vpidr_el2, x2 260 261 /* Load the Virtualization Multiprocess ID Register */ 262 mrs x2, mpidr_el1 263 msr vmpidr_el2, x2 264 265 /* Set the bits that need to be 1 in sctlr_el1 */ 266 ldr x2, .Lsctlr_res1 267 msr sctlr_el1, x2 268 269 /* Don't trap to EL2 for exceptions */ 270 mov x2, #CPTR_RES1 271 msr cptr_el2, x2 272 273 /* Don't trap to EL2 for CP15 traps */ 274 msr hstr_el2, xzr 275 276 /* Enable access to the physical timers at EL1 */ 277 mrs x2, cnthctl_el2 278 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 279 msr cnthctl_el2, x2 280 281 /* Set the counter offset to a known value */ 282 msr cntvoff_el2, xzr 283 284 /* Hypervisor trap functions */ 285 adr x2, hyp_vectors 286 msr vbar_el2, x2 287 288 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 289 msr spsr_el2, x2 290 291 /* Configure GICv3 CPU interface */ 292 mrs x2, id_aa64pfr0_el1 293 /* Extract GIC bits from the register */ 294 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 295 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 296 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 297 b.ne 2f 298 299 mrs x2, icc_sre_el2 300 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 301 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 302 msr icc_sre_el2, x2 3032: 304 305 /* Set the address to return to our return address */ 306 msr elr_el2, x30 307 isb 308 309 eret 310 311 .align 3 312.Lsctlr_res1: 313 .quad SCTLR_RES1 314 315#define VECT_EMPTY \ 316 .align 7; \ 317 1: b 1b 318 319 .align 11 320hyp_vectors: 321 VECT_EMPTY /* Synchronous EL2t */ 322 VECT_EMPTY /* IRQ EL2t */ 323 VECT_EMPTY /* FIQ EL2t */ 324 VECT_EMPTY /* Error EL2t */ 325 326 VECT_EMPTY /* Synchronous EL2h */ 327 VECT_EMPTY /* IRQ EL2h */ 328 VECT_EMPTY /* FIQ EL2h */ 329 VECT_EMPTY /* Error EL2h */ 330 331 VECT_EMPTY /* Synchronous 64-bit EL1 */ 332 VECT_EMPTY /* IRQ 64-bit EL1 */ 333 VECT_EMPTY /* FIQ 64-bit EL1 */ 334 VECT_EMPTY /* Error 64-bit EL1 */ 335 336 VECT_EMPTY /* Synchronous 32-bit EL1 */ 337 VECT_EMPTY /* IRQ 32-bit EL1 */ 338 VECT_EMPTY /* FIQ 32-bit EL1 */ 339 VECT_EMPTY /* Error 32-bit EL1 */ 340 341/* 342 * Get the delta between the physical address we were loaded to and the 343 * virtual address we expect to run from. This is used when building the 344 * initial page table. 345 */ 346get_virt_delta: 347 /* Load the physical address of virt_map */ 348 adr x29, virt_map 349 /* Load the virtual address of virt_map stored in virt_map */ 350 ldr x28, [x29] 351 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 352 sub x29, x29, x28 353 /* Find the load address for the kernel */ 354 mov x28, #(KERNBASE) 355 add x28, x28, x29 356 ret 357 358 .align 3 359virt_map: 360 .quad virt_map 361 362/* 363 * This builds the page tables containing the identity map, and the kernel 364 * virtual map. 365 * 366 * It relys on: 367 * We were loaded to an address that is on a 2MiB boundary 368 * All the memory must not cross a 1GiB boundaty 369 * x28 contains the physical address we were loaded from 370 * 371 * TODO: This is out of date. 372 * There are at least 5 pages before that address for the page tables 373 * The pages used are: 374 * - The Kernel L2 table 375 * - The Kernel L1 table 376 * - The Kernel L0 table (TTBR1) 377 * - The identity (PA = VA) L1 table 378 * - The identity (PA = VA) L0 table (TTBR0) 379 * - The DMAP L1 tables 380 */ 381create_pagetables: 382 /* Save the Link register */ 383 mov x5, x30 384 385 /* Clean the page table */ 386 adr x6, pagetable 387 mov x26, x6 388 adr x27, pagetable_end 3891: 390 stp xzr, xzr, [x6], #16 391 stp xzr, xzr, [x6], #16 392 stp xzr, xzr, [x6], #16 393 stp xzr, xzr, [x6], #16 394 cmp x6, x27 395 b.lo 1b 396 397 /* 398 * Build the TTBR1 maps. 399 */ 400 401 /* Find the size of the kernel */ 402 mov x6, #(KERNBASE) 403 404#if defined(LINUX_BOOT_ABI) 405 /* X19 is used as 'map FDT data' flag */ 406 mov x19, xzr 407 408 /* No modules or FDT pointer ? */ 409 cbz x0, booti_no_fdt 410 411 /* Test if modulep points to modules descriptor or to FDT */ 412 ldr w8, [x0] 413 ldr w7, =FDT_MAGIC 414 cmp w7, w8 415 b.eq booti_fdt 416#endif 417 418 /* Booted with modules pointer */ 419 /* Find modulep - begin */ 420 sub x8, x0, x6 421 /* Add two 2MiB pages for the module data and round up */ 422 ldr x7, =(3 * L2_SIZE - 1) 423 add x8, x8, x7 424 b common 425 426#if defined(LINUX_BOOT_ABI) 427booti_fdt: 428 /* Booted by U-Boot booti with FDT data */ 429 /* Set 'map FDT data' flag */ 430 mov x19, #1 431 432booti_no_fdt: 433 /* Booted by U-Boot booti without FTD data */ 434 /* Find the end - begin */ 435 ldr x7, .Lend 436 sub x8, x7, x6 437 438 /* 439 * Add one 2MiB page for copy of FDT data (maximum FDT size), 440 * one for metadata and round up 441 */ 442 ldr x7, =(3 * L2_SIZE - 1) 443 add x8, x8, x7 444#endif 445 446common: 447 /* Get the number of l2 pages to allocate, rounded down */ 448 lsr x10, x8, #(L2_SHIFT) 449 450 /* Create the kernel space L2 table */ 451 mov x6, x26 452 mov x7, #VM_MEMATTR_WRITE_BACK 453 mov x8, #(KERNBASE & L2_BLOCK_MASK) 454 mov x9, x28 455 bl build_l2_block_pagetable 456 457 /* Move to the l1 table */ 458 add x26, x26, #PAGE_SIZE 459 460 /* Link the l1 -> l2 table */ 461 mov x9, x6 462 mov x6, x26 463 bl link_l1_pagetable 464 465 /* Move to the l0 table */ 466 add x24, x26, #PAGE_SIZE 467 468 /* Link the l0 -> l1 table */ 469 mov x9, x6 470 mov x6, x24 471 mov x10, #1 472 bl link_l0_pagetable 473 474 /* Link the DMAP tables */ 475 ldr x8, =DMAP_MIN_ADDRESS 476 adr x9, pagetable_dmap; 477 mov x10, #DMAP_TABLES 478 bl link_l0_pagetable 479 480 /* 481 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 482 * They are only needed early on, so the VA = PA map is uncached. 483 */ 484 add x27, x24, #PAGE_SIZE 485 486 mov x6, x27 /* The initial page table */ 487#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 488 /* Create a table for the UART */ 489 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 490 mov x8, #(SOCDEV_VA) /* VA start */ 491 mov x9, #(SOCDEV_PA) /* PA start */ 492 mov x10, #1 493 bl build_l1_block_pagetable 494#endif 495 496#if defined(LINUX_BOOT_ABI) 497 /* Map FDT data ? */ 498 cbz x19, 1f 499 500 /* Create the identity mapping for FDT data (2 MiB max) */ 501 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_UNCACHEABLE)) 502 mov x9, x0 503 mov x8, x0 /* VA start (== PA start) */ 504 mov x10, #1 505 bl build_l1_block_pagetable 506 5071: 508#endif 509 510 /* Create the VA = PA map */ 511 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_UNCACHEABLE)) 512 mov x9, x27 513 mov x8, x9 /* VA start (== PA start) */ 514 mov x10, #1 515 bl build_l1_block_pagetable 516 517 /* Move to the l0 table */ 518 add x27, x27, #PAGE_SIZE 519 520 /* Link the l0 -> l1 table */ 521 mov x9, x6 522 mov x6, x27 523 mov x10, #1 524 bl link_l0_pagetable 525 526 /* Restore the Link register */ 527 mov x30, x5 528 ret 529 530/* 531 * Builds an L0 -> L1 table descriptor 532 * 533 * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 534 * within it by build_l1_block_pagetable. 535 * 536 * x6 = L0 table 537 * x8 = Virtual Address 538 * x9 = L1 PA (trashed) 539 * x10 = Entry count 540 * x11, x12 and x13 are trashed 541 */ 542link_l0_pagetable: 543 /* 544 * Link an L0 -> L1 table entry. 545 */ 546 /* Find the table index */ 547 lsr x11, x8, #L0_SHIFT 548 and x11, x11, #L0_ADDR_MASK 549 550 /* Build the L0 block entry */ 551 mov x12, #L0_TABLE 552 553 /* Only use the output address bits */ 554 lsr x9, x9, #PAGE_SHIFT 5551: orr x13, x12, x9, lsl #PAGE_SHIFT 556 557 /* Store the entry */ 558 str x13, [x6, x11, lsl #3] 559 560 sub x10, x10, #1 561 add x11, x11, #1 562 add x9, x9, #1 563 cbnz x10, 1b 564 565 ret 566 567/* 568 * Builds an L1 -> L2 table descriptor 569 * 570 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 571 * within it by build_l2_block_pagetable. 572 * 573 * x6 = L1 table 574 * x8 = Virtual Address 575 * x9 = L2 PA (trashed) 576 * x11, x12 and x13 are trashed 577 */ 578link_l1_pagetable: 579 /* 580 * Link an L1 -> L2 table entry. 581 */ 582 /* Find the table index */ 583 lsr x11, x8, #L1_SHIFT 584 and x11, x11, #Ln_ADDR_MASK 585 586 /* Build the L1 block entry */ 587 mov x12, #L1_TABLE 588 589 /* Only use the output address bits */ 590 lsr x9, x9, #PAGE_SHIFT 591 orr x13, x12, x9, lsl #PAGE_SHIFT 592 593 /* Store the entry */ 594 str x13, [x6, x11, lsl #3] 595 596 ret 597 598/* 599 * Builds count 1 GiB page table entry 600 * x6 = L1 table 601 * x7 = Variable lower block attributes 602 * x8 = VA start 603 * x9 = PA start (trashed) 604 * x10 = Entry count 605 * x11, x12 and x13 are trashed 606 */ 607build_l1_block_pagetable: 608 /* 609 * Build the L1 table entry. 610 */ 611 /* Find the table index */ 612 lsr x11, x8, #L1_SHIFT 613 and x11, x11, #Ln_ADDR_MASK 614 615 /* Build the L1 block entry */ 616 orr x12, x7, #L1_BLOCK 617 orr x12, x12, #(ATTR_AF) 618#ifdef SMP 619 orr x12, x12, ATTR_SH(ATTR_SH_IS) 620#endif 621 622 /* Only use the output address bits */ 623 lsr x9, x9, #L1_SHIFT 624 625 /* Set the physical address for this virtual address */ 6261: orr x13, x12, x9, lsl #L1_SHIFT 627 628 /* Store the entry */ 629 str x13, [x6, x11, lsl #3] 630 631 sub x10, x10, #1 632 add x11, x11, #1 633 add x9, x9, #1 634 cbnz x10, 1b 635 636 ret 637 638/* 639 * Builds count 2 MiB page table entry 640 * x6 = L2 table 641 * x7 = Type (0 = Device, 1 = Normal) 642 * x8 = VA start 643 * x9 = PA start (trashed) 644 * x10 = Entry count 645 * x11, x12 and x13 are trashed 646 */ 647build_l2_block_pagetable: 648 /* 649 * Build the L2 table entry. 650 */ 651 /* Find the table index */ 652 lsr x11, x8, #L2_SHIFT 653 and x11, x11, #Ln_ADDR_MASK 654 655 /* Build the L2 block entry */ 656 lsl x12, x7, #2 657 orr x12, x12, #L2_BLOCK 658 orr x12, x12, #(ATTR_AF) 659 orr x12, x12, #(ATTR_S1_UXN) 660#ifdef SMP 661 orr x12, x12, ATTR_SH(ATTR_SH_IS) 662#endif 663 664 /* Only use the output address bits */ 665 lsr x9, x9, #L2_SHIFT 666 667 /* Set the physical address for this virtual address */ 6681: orr x13, x12, x9, lsl #L2_SHIFT 669 670 /* Store the entry */ 671 str x13, [x6, x11, lsl #3] 672 673 sub x10, x10, #1 674 add x11, x11, #1 675 add x9, x9, #1 676 cbnz x10, 1b 677 678 ret 679 680start_mmu: 681 dsb sy 682 683 /* Load the exception vectors */ 684 ldr x2, =exception_vectors 685 msr vbar_el1, x2 686 687 /* Load ttbr0 and ttbr1 */ 688 msr ttbr0_el1, x27 689 msr ttbr1_el1, x24 690 isb 691 692 /* Clear the Monitor Debug System control register */ 693 msr mdscr_el1, xzr 694 695 /* Invalidate the TLB */ 696 tlbi vmalle1is 697 dsb ish 698 isb 699 700 ldr x2, mair 701 msr mair_el1, x2 702 703 /* 704 * Setup TCR according to the PARange and ASIDBits fields 705 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 706 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 707 * to 1 only if the ASIDBits field equals 0b0010. 708 */ 709 ldr x2, tcr 710 mrs x3, id_aa64mmfr0_el1 711 712 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 713 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 714 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 715 716 /* Check if the HW supports 16 bit ASIDS */ 717 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 718 /* If so x3 == 1, else x3 == 0 */ 719 cset x3, eq 720 /* Set TCR.AS with x3 */ 721 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 722 723 /* 724 * Check if the HW supports access flag and dirty state updates, 725 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 726 */ 727 mrs x3, id_aa64mmfr1_el1 728 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 729 cmp x3, #1 730 b.ne 1f 731 orr x2, x2, #(TCR_HA) 732 b 2f 7331: 734 cmp x3, #2 735 b.ne 2f 736 orr x2, x2, #(TCR_HA | TCR_HD) 7372: 738 msr tcr_el1, x2 739 740 /* 741 * Setup SCTLR. 742 */ 743 ldr x2, sctlr_set 744 ldr x3, sctlr_clear 745 mrs x1, sctlr_el1 746 bic x1, x1, x3 /* Clear the required bits */ 747 orr x1, x1, x2 /* Set the required bits */ 748 msr sctlr_el1, x1 749 isb 750 751 ret 752 753 .align 3 754mair: 755 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | \ 756 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 757 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 758 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) 759tcr: 760 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG1_4K | \ 761 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 762sctlr_set: 763 /* Bits to set */ 764 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 765 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 766 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 767 SCTLR_M | SCTLR_CP15BEN) 768sctlr_clear: 769 /* Bits to clear */ 770 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 771 SCTLR_ITD | SCTLR_A) 772 773 .globl abort 774abort: 775 b abort 776 777 //.section .init_pagetable 778 .align 12 /* 4KiB aligned */ 779 /* 780 * 6 initial tables (in the following order): 781 * L2 for kernel (High addresses) 782 * L1 for kernel 783 * L0 for kernel 784 * L1 bootstrap for user (Low addresses) 785 * L0 bootstrap for user 786 * L0 for user 787 */ 788pagetable: 789 .space PAGE_SIZE 790pagetable_l1_ttbr1: 791 .space PAGE_SIZE 792pagetable_l0_ttbr1: 793 .space PAGE_SIZE 794pagetable_l1_ttbr0_bootstrap: 795 .space PAGE_SIZE 796pagetable_l0_ttbr0_boostrap: 797 .space PAGE_SIZE 798pagetable_l0_ttbr0: 799 .space PAGE_SIZE 800 801 .globl pagetable_dmap 802pagetable_dmap: 803 .space PAGE_SIZE * DMAP_TABLES 804pagetable_end: 805 806el2_pagetable: 807 .space PAGE_SIZE 808 809 .globl init_pt_va 810init_pt_va: 811 .quad pagetable /* XXX: Keep page tables VA */ 812 813 .align 4 814initstack: 815 .space (PAGE_SIZE * KSTACK_PAGES) 816initstack_end: 817 818 819ENTRY(sigcode) 820 mov x0, sp 821 add x0, x0, #SF_UC 822 8231: 824 mov x8, #SYS_sigreturn 825 svc 0 826 827 /* sigreturn failed, exit */ 828 mov x8, #SYS_exit 829 svc 0 830 831 b 1b 832END(sigcode) 833 /* This may be copied to the stack, keep it 16-byte aligned */ 834 .align 3 835esigcode: 836 837 .data 838 .align 3 839 .global szsigcode 840szsigcode: 841 .quad esigcode - sigcode 842 843ENTRY(aarch32_sigcode) 844 .word 0xe1a0000d // mov r0, sp 845 .word 0xe2800040 // add r0, r0, #SIGF_UC 846 .word 0xe59f700c // ldr r7, [pc, #12] 847 .word 0xef000000 // swi #0 848 .word 0xe59f7008 // ldr r7, [pc, #8] 849 .word 0xef000000 // swi #0 850 .word 0xeafffffa // b . - 16 851END(aarch32_sigcode) 852 .word SYS_sigreturn 853 .word SYS_exit 854 .align 3 855aarch32_esigcode: 856 .data 857 .global sz_aarch32_sigcode 858sz_aarch32_sigcode: 859 .quad aarch32_esigcode - aarch32_sigcode 860