1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) 42 43 .globl kernbase 44 .set kernbase, KERNBASE 45 46/* 47 * We assume: 48 * MMU on with an identity map, or off 49 * D-Cache: off 50 * I-Cache: on or off 51 * We are loaded at a 2MiB aligned address 52 */ 53 54 .text 55 .globl _start 56_start: 57 /* Drop to EL1 */ 58 bl drop_to_el1 59 60 /* 61 * Disable the MMU. We may have entered the kernel with it on and 62 * will need to update the tables later. If this has been set up 63 * with anything other than a VA == PA map then this will fail, 64 * but in this case the code to find where we are running from 65 * would have also failed. 66 */ 67 dsb sy 68 mrs x2, sctlr_el1 69 bic x2, x2, SCTLR_M 70 msr sctlr_el1, x2 71 isb 72 73 /* Set the context id */ 74 msr contextidr_el1, xzr 75 76 /* Get the virt -> phys offset */ 77 bl get_virt_delta 78 79 /* 80 * At this point: 81 * x29 = PA - VA 82 * x28 = Our physical load address 83 */ 84 85 /* Create the page tables */ 86 bl create_pagetables 87 88 /* 89 * At this point: 90 * x27 = TTBR0 table 91 * x26 = Kernel L1 table 92 * x24 = TTBR1 table 93 */ 94 95 /* Enable the mmu */ 96 bl start_mmu 97 98 /* Jump to the virtual address space */ 99 ldr x15, .Lvirtdone 100 br x15 101 102virtdone: 103 /* Set up the stack */ 104 adr x25, initstack_end 105 mov sp, x25 106 sub sp, sp, #PCB_SIZE 107 108 /* Zero the BSS */ 109 ldr x15, .Lbss 110 ldr x14, .Lend 1111: 112 str xzr, [x15], #8 113 cmp x15, x14 114 b.lo 1b 115 116 /* Backup the module pointer */ 117 mov x1, x0 118 119 /* Make the page table base a virtual address */ 120 sub x26, x26, x29 121 sub x24, x24, x29 122 123 sub sp, sp, #(64 * 4) 124 mov x0, sp 125 126 /* Degate the delda so it is VA -> PA */ 127 neg x29, x29 128 129 str x1, [x0] /* modulep */ 130 str x26, [x0, 8] /* kern_l1pt */ 131 str x29, [x0, 16] /* kern_delta */ 132 adr x25, initstack 133 str x25, [x0, 24] /* kern_stack */ 134 str x24, [x0, 32] /* kern_l0pt */ 135 136 /* trace back starts here */ 137 mov fp, #0 138 /* Branch to C code */ 139 bl initarm 140 bl mi_startup 141 142 /* We should not get here */ 143 brk 0 144 145 .align 3 146.Lvirtdone: 147 .quad virtdone 148.Lbss: 149 .quad __bss_start 150.Lend: 151 .quad _end 152 153#ifdef SMP 154/* 155 * mpentry(unsigned long) 156 * 157 * Called by a core when it is being brought online. 158 * The data in x0 is passed straight to init_secondary. 159 */ 160ENTRY(mpentry) 161 /* Disable interrupts */ 162 msr daifset, #2 163 164 /* Drop to EL1 */ 165 bl drop_to_el1 166 167 /* Set the context id */ 168 msr contextidr_el1, xzr 169 170 /* Load the kernel page table */ 171 adr x24, pagetable_l0_ttbr1 172 /* Load the identity page table */ 173 adr x27, pagetable_l0_ttbr0 174 175 /* Enable the mmu */ 176 bl start_mmu 177 178 /* Jump to the virtual address space */ 179 ldr x15, =mp_virtdone 180 br x15 181 182mp_virtdone: 183 ldr x4, =secondary_stacks 184 mov x5, #(PAGE_SIZE * KSTACK_PAGES) 185 mul x5, x0, x5 186 add sp, x4, x5 187 188 b init_secondary 189END(mpentry) 190#endif 191 192/* 193 * If we are started in EL2, configure the required hypervisor 194 * registers and drop to EL1. 195 */ 196drop_to_el1: 197 mrs x1, CurrentEL 198 lsr x1, x1, #2 199 cmp x1, #0x2 200 b.eq 1f 201 ret 2021: 203 /* Configure the Hypervisor */ 204 mov x2, #(HCR_RW) 205 msr hcr_el2, x2 206 207 /* Load the Virtualization Process ID Register */ 208 mrs x2, midr_el1 209 msr vpidr_el2, x2 210 211 /* Load the Virtualization Multiprocess ID Register */ 212 mrs x2, mpidr_el1 213 msr vmpidr_el2, x2 214 215 /* Set the bits that need to be 1 in sctlr_el1 */ 216 ldr x2, .Lsctlr_res1 217 msr sctlr_el1, x2 218 219 /* Don't trap to EL2 for exceptions */ 220 mov x2, #CPTR_RES1 221 msr cptr_el2, x2 222 223 /* Don't trap to EL2 for CP15 traps */ 224 msr hstr_el2, xzr 225 226 /* Enable access to the physical timers at EL1 */ 227 mrs x2, cnthctl_el2 228 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 229 msr cnthctl_el2, x2 230 231 /* Set the counter offset to a known value */ 232 msr cntvoff_el2, xzr 233 234 /* Hypervisor trap functions */ 235 adr x2, hyp_vectors 236 msr vbar_el2, x2 237 238 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 239 msr spsr_el2, x2 240 241 /* Configure GICv3 CPU interface */ 242 mrs x2, id_aa64pfr0_el1 243 /* Extract GIC bits from the register */ 244 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 245 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 246 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 247 b.ne 2f 248 249 mrs x2, icc_sre_el2 250 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 251 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 252 msr icc_sre_el2, x2 2532: 254 255 /* Set the address to return to our return address */ 256 msr elr_el2, x30 257 isb 258 259 eret 260 261 .align 3 262.Lsctlr_res1: 263 .quad SCTLR_RES1 264 265#define VECT_EMPTY \ 266 .align 7; \ 267 1: b 1b 268 269 .align 11 270hyp_vectors: 271 VECT_EMPTY /* Synchronous EL2t */ 272 VECT_EMPTY /* IRQ EL2t */ 273 VECT_EMPTY /* FIQ EL2t */ 274 VECT_EMPTY /* Error EL2t */ 275 276 VECT_EMPTY /* Synchronous EL2h */ 277 VECT_EMPTY /* IRQ EL2h */ 278 VECT_EMPTY /* FIQ EL2h */ 279 VECT_EMPTY /* Error EL2h */ 280 281 VECT_EMPTY /* Synchronous 64-bit EL1 */ 282 VECT_EMPTY /* IRQ 64-bit EL1 */ 283 VECT_EMPTY /* FIQ 64-bit EL1 */ 284 VECT_EMPTY /* Error 64-bit EL1 */ 285 286 VECT_EMPTY /* Synchronous 32-bit EL1 */ 287 VECT_EMPTY /* IRQ 32-bit EL1 */ 288 VECT_EMPTY /* FIQ 32-bit EL1 */ 289 VECT_EMPTY /* Error 32-bit EL1 */ 290 291/* 292 * Get the delta between the physical address we were loaded to and the 293 * virtual address we expect to run from. This is used when building the 294 * initial page table. 295 */ 296get_virt_delta: 297 /* Load the physical address of virt_map */ 298 adr x29, virt_map 299 /* Load the virtual address of virt_map stored in virt_map */ 300 ldr x28, [x29] 301 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 302 sub x29, x29, x28 303 /* Find the load address for the kernel */ 304 mov x28, #(KERNBASE) 305 add x28, x28, x29 306 ret 307 308 .align 3 309virt_map: 310 .quad virt_map 311 312/* 313 * This builds the page tables containing the identity map, and the kernel 314 * virtual map. 315 * 316 * It relys on: 317 * We were loaded to an address that is on a 2MiB boundary 318 * All the memory must not cross a 1GiB boundaty 319 * x28 contains the physical address we were loaded from 320 * 321 * TODO: This is out of date. 322 * There are at least 5 pages before that address for the page tables 323 * The pages used are: 324 * - The Kernel L2 table 325 * - The Kernel L1 table 326 * - The Kernel L0 table (TTBR1) 327 * - The identity (PA = VA) L1 table 328 * - The identity (PA = VA) L0 table (TTBR0) 329 * - The DMAP L1 tables 330 */ 331create_pagetables: 332 /* Save the Link register */ 333 mov x5, x30 334 335 /* Clean the page table */ 336 adr x6, pagetable 337 mov x26, x6 338 adr x27, pagetable_end 3391: 340 stp xzr, xzr, [x6], #16 341 stp xzr, xzr, [x6], #16 342 stp xzr, xzr, [x6], #16 343 stp xzr, xzr, [x6], #16 344 cmp x6, x27 345 b.lo 1b 346 347 /* 348 * Build the TTBR1 maps. 349 */ 350 351 /* Find the size of the kernel */ 352 mov x6, #(KERNBASE) 353 /* Find modulep - begin */ 354 sub x8, x0, x6 355 /* Add two 2MiB pages for the module data and round up */ 356 ldr x7, =(3 * L2_SIZE - 1) 357 add x8, x8, x7 358 /* Get the number of l2 pages to allocate, rounded down */ 359 lsr x10, x8, #(L2_SHIFT) 360 361 /* Create the kernel space L2 table */ 362 mov x6, x26 363 mov x7, #VM_MEMATTR_WRITE_BACK 364 mov x8, #(KERNBASE & L2_BLOCK_MASK) 365 mov x9, x28 366 bl build_l2_block_pagetable 367 368 /* Move to the l1 table */ 369 add x26, x26, #PAGE_SIZE 370 371 /* Link the l1 -> l2 table */ 372 mov x9, x6 373 mov x6, x26 374 bl link_l1_pagetable 375 376 /* Move to the l0 table */ 377 add x24, x26, #PAGE_SIZE 378 379 /* Link the l0 -> l1 table */ 380 mov x9, x6 381 mov x6, x24 382 mov x10, #1 383 bl link_l0_pagetable 384 385 /* Link the DMAP tables */ 386 ldr x8, =DMAP_MIN_ADDRESS 387 adr x9, pagetable_dmap; 388 mov x10, #DMAP_TABLES 389 bl link_l0_pagetable 390 391 /* 392 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_nG. 393 * They are only needed early on, so the VA = PA map is uncached. 394 */ 395 add x27, x24, #PAGE_SIZE 396 397 mov x6, x27 /* The initial page table */ 398#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 399 /* Create a table for the UART */ 400 mov x7, #(ATTR_nG | ATTR_IDX(VM_MEMATTR_DEVICE)) 401 mov x8, #(SOCDEV_VA) /* VA start */ 402 mov x9, #(SOCDEV_PA) /* PA start */ 403 mov x10, #1 404 bl build_l1_block_pagetable 405#endif 406 407 /* 408 * Create the VA = PA map 409 */ 410 mov x7, #(ATTR_nG | ATTR_IDX(VM_MEMATTR_UNCACHEABLE)) 411 mov x9, x27 412 mov x8, x9 /* VA start (== PA start) */ 413 mov x10, #1 414 bl build_l1_block_pagetable 415 416 /* Move to the l0 table */ 417 add x27, x27, #PAGE_SIZE 418 419 /* Link the l0 -> l1 table */ 420 mov x9, x6 421 mov x6, x27 422 mov x10, #1 423 bl link_l0_pagetable 424 425 /* Restore the Link register */ 426 mov x30, x5 427 ret 428 429/* 430 * Builds an L0 -> L1 table descriptor 431 * 432 * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 433 * within it by build_l1_block_pagetable. 434 * 435 * x6 = L0 table 436 * x8 = Virtual Address 437 * x9 = L1 PA (trashed) 438 * x10 = Entry count 439 * x11, x12 and x13 are trashed 440 */ 441link_l0_pagetable: 442 /* 443 * Link an L0 -> L1 table entry. 444 */ 445 /* Find the table index */ 446 lsr x11, x8, #L0_SHIFT 447 and x11, x11, #L0_ADDR_MASK 448 449 /* Build the L0 block entry */ 450 mov x12, #L0_TABLE 451 452 /* Only use the output address bits */ 453 lsr x9, x9, #PAGE_SHIFT 4541: orr x13, x12, x9, lsl #PAGE_SHIFT 455 456 /* Store the entry */ 457 str x13, [x6, x11, lsl #3] 458 459 sub x10, x10, #1 460 add x11, x11, #1 461 add x9, x9, #1 462 cbnz x10, 1b 463 464 ret 465 466/* 467 * Builds an L1 -> L2 table descriptor 468 * 469 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 470 * within it by build_l2_block_pagetable. 471 * 472 * x6 = L1 table 473 * x8 = Virtual Address 474 * x9 = L2 PA (trashed) 475 * x11, x12 and x13 are trashed 476 */ 477link_l1_pagetable: 478 /* 479 * Link an L1 -> L2 table entry. 480 */ 481 /* Find the table index */ 482 lsr x11, x8, #L1_SHIFT 483 and x11, x11, #Ln_ADDR_MASK 484 485 /* Build the L1 block entry */ 486 mov x12, #L1_TABLE 487 488 /* Only use the output address bits */ 489 lsr x9, x9, #PAGE_SHIFT 490 orr x13, x12, x9, lsl #PAGE_SHIFT 491 492 /* Store the entry */ 493 str x13, [x6, x11, lsl #3] 494 495 ret 496 497/* 498 * Builds count 1 GiB page table entry 499 * x6 = L1 table 500 * x7 = Variable lower block attributes 501 * x8 = VA start 502 * x9 = PA start (trashed) 503 * x10 = Entry count 504 * x11, x12 and x13 are trashed 505 */ 506build_l1_block_pagetable: 507 /* 508 * Build the L1 table entry. 509 */ 510 /* Find the table index */ 511 lsr x11, x8, #L1_SHIFT 512 and x11, x11, #Ln_ADDR_MASK 513 514 /* Build the L1 block entry */ 515 orr x12, x7, #L1_BLOCK 516 orr x12, x12, #(ATTR_AF) 517#ifdef SMP 518 orr x12, x12, ATTR_SH(ATTR_SH_IS) 519#endif 520 521 /* Only use the output address bits */ 522 lsr x9, x9, #L1_SHIFT 523 524 /* Set the physical address for this virtual address */ 5251: orr x13, x12, x9, lsl #L1_SHIFT 526 527 /* Store the entry */ 528 str x13, [x6, x11, lsl #3] 529 530 sub x10, x10, #1 531 add x11, x11, #1 532 add x9, x9, #1 533 cbnz x10, 1b 534 535 ret 536 537/* 538 * Builds count 2 MiB page table entry 539 * x6 = L2 table 540 * x7 = Type (0 = Device, 1 = Normal) 541 * x8 = VA start 542 * x9 = PA start (trashed) 543 * x10 = Entry count 544 * x11, x12 and x13 are trashed 545 */ 546build_l2_block_pagetable: 547 /* 548 * Build the L2 table entry. 549 */ 550 /* Find the table index */ 551 lsr x11, x8, #L2_SHIFT 552 and x11, x11, #Ln_ADDR_MASK 553 554 /* Build the L2 block entry */ 555 lsl x12, x7, #2 556 orr x12, x12, #L2_BLOCK 557 orr x12, x12, #(ATTR_AF) 558 orr x12, x12, #(ATTR_UXN) 559#ifdef SMP 560 orr x12, x12, ATTR_SH(ATTR_SH_IS) 561#endif 562 563 /* Only use the output address bits */ 564 lsr x9, x9, #L2_SHIFT 565 566 /* Set the physical address for this virtual address */ 5671: orr x13, x12, x9, lsl #L2_SHIFT 568 569 /* Store the entry */ 570 str x13, [x6, x11, lsl #3] 571 572 sub x10, x10, #1 573 add x11, x11, #1 574 add x9, x9, #1 575 cbnz x10, 1b 576 577 ret 578 579start_mmu: 580 dsb sy 581 582 /* Load the exception vectors */ 583 ldr x2, =exception_vectors 584 msr vbar_el1, x2 585 586 /* Load ttbr0 and ttbr1 */ 587 msr ttbr0_el1, x27 588 msr ttbr1_el1, x24 589 isb 590 591 /* Clear the Monitor Debug System control register */ 592 msr mdscr_el1, xzr 593 594 /* Invalidate the TLB */ 595 tlbi vmalle1is 596 597 ldr x2, mair 598 msr mair_el1, x2 599 600 /* 601 * Setup TCR according to the PARange and ASIDBits fields 602 * from ID_AA64MMFR0_EL1. More precisely, set TCR_EL1.AS 603 * to 1 only if the ASIDBits field equals 0b0010. 604 */ 605 ldr x2, tcr 606 mrs x3, id_aa64mmfr0_el1 607 bfi x2, x3, #32, #3 608 and x3, x3, #0xF0 609 cmp x3, #0x20 610 cset x3, eq 611 bfi x2, x3, #36, #1 612 msr tcr_el1, x2 613 614 /* Setup SCTLR */ 615 ldr x2, sctlr_set 616 ldr x3, sctlr_clear 617 mrs x1, sctlr_el1 618 bic x1, x1, x3 /* Clear the required bits */ 619 orr x1, x1, x2 /* Set the required bits */ 620 msr sctlr_el1, x1 621 isb 622 623 ret 624 625 .align 3 626mair: 627 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE) | \ 628 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 629 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 630 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) 631tcr: 632 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG1_4K | \ 633 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 634sctlr_set: 635 /* Bits to set */ 636 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 637 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 638 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 639 SCTLR_M | SCTLR_CP15BEN) 640sctlr_clear: 641 /* Bits to clear */ 642 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 643 SCTLR_ITD | SCTLR_A) 644 645 .globl abort 646abort: 647 b abort 648 649 //.section .init_pagetable 650 .align 12 /* 4KiB aligned */ 651 /* 652 * 3 initial tables (in the following order): 653 * L2 for kernel (High addresses) 654 * L1 for kernel 655 * L1 for user (Low addresses) 656 */ 657pagetable: 658 .space PAGE_SIZE 659pagetable_l1_ttbr1: 660 .space PAGE_SIZE 661pagetable_l0_ttbr1: 662 .space PAGE_SIZE 663pagetable_l1_ttbr0: 664 .space PAGE_SIZE 665pagetable_l0_ttbr0: 666 .space PAGE_SIZE 667 668 .globl pagetable_dmap 669pagetable_dmap: 670 .space PAGE_SIZE * DMAP_TABLES 671pagetable_end: 672 673el2_pagetable: 674 .space PAGE_SIZE 675 676 .globl init_pt_va 677init_pt_va: 678 .quad pagetable /* XXX: Keep page tables VA */ 679 680 .align 4 681initstack: 682 .space (PAGE_SIZE * KSTACK_PAGES) 683initstack_end: 684 685 686ENTRY(sigcode) 687 mov x0, sp 688 add x0, x0, #SF_UC 689 6901: 691 mov x8, #SYS_sigreturn 692 svc 0 693 694 /* sigreturn failed, exit */ 695 mov x8, #SYS_exit 696 svc 0 697 698 b 1b 699END(sigcode) 700 /* This may be copied to the stack, keep it 16-byte aligned */ 701 .align 3 702esigcode: 703 704 .data 705 .align 3 706 .global szsigcode 707szsigcode: 708 .quad esigcode - sigcode 709 710ENTRY(aarch32_sigcode) 711 .word 0xe1a0000d // mov r0, sp 712 .word 0xe2800040 // add r0, r0, #SIGF_UC 713 .word 0xe59f700c // ldr r7, [pc, #12] 714 .word 0xef000000 // swi #0 715 .word 0xe59f7008 // ldr r7, [pc, #8] 716 .word 0xef000000 // swi #0 717 .word 0xeafffffa // b . - 16 718END(aarch32_sigcode) 719 .word SYS_sigreturn 720 .word SYS_exit 721 .align 3 722aarch32_esigcode: 723 .data 724 .global sz_aarch32_sigcode 725sz_aarch32_sigcode: 726 .quad aarch32_esigcode - aarch32_sigcode 727