1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vmparam.h> 38 39#define VIRT_BITS 48 40#define DMAP_TABLES ((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT) 41 42 .globl kernbase 43 .set kernbase, KERNBASE 44 45#define DEVICE_MEM 0 46#define NORMAL_UNCACHED 1 47#define NORMAL_MEM 2 48 49/* 50 * We assume: 51 * MMU on with an identity map, or off 52 * D-Cache: off 53 * I-Cache: on or off 54 * We are loaded at a 2MiB aligned address 55 */ 56 57 .text 58 .globl _start 59_start: 60 /* Drop to EL1 */ 61 bl drop_to_el1 62 63 /* 64 * Disable the MMU. We may have entered the kernel with it on and 65 * will need to update the tables later. If this has been set up 66 * with anything other than a VA == PA map then this will fail, 67 * but in this case the code to find where we are running from 68 * would have also failed. 69 */ 70 dsb sy 71 mrs x2, sctlr_el1 72 bic x2, x2, SCTLR_M 73 msr sctlr_el1, x2 74 isb 75 76 /* Set the context id */ 77 msr contextidr_el1, xzr 78 79 /* Get the virt -> phys offset */ 80 bl get_virt_delta 81 82 /* 83 * At this point: 84 * x29 = PA - VA 85 * x28 = Our physical load address 86 */ 87 88 /* Create the page tables */ 89 bl create_pagetables 90 91 /* 92 * At this point: 93 * x27 = TTBR0 table 94 * x26 = Kernel L1 table 95 * x24 = TTBR1 table 96 */ 97 98 /* Enable the mmu */ 99 bl start_mmu 100 101 /* Jump to the virtual address space */ 102 ldr x15, .Lvirtdone 103 br x15 104 105virtdone: 106 /* Set up the stack */ 107 adr x25, initstack_end 108 mov sp, x25 109 sub sp, sp, #PCB_SIZE 110 111 /* Zero the BSS */ 112 ldr x15, .Lbss 113 ldr x14, .Lend 1141: 115 str xzr, [x15], #8 116 cmp x15, x14 117 b.lo 1b 118 119 /* Backup the module pointer */ 120 mov x1, x0 121 122 /* Make the page table base a virtual address */ 123 sub x26, x26, x29 124 sub x24, x24, x29 125 126 sub sp, sp, #(64 * 4) 127 mov x0, sp 128 129 /* Degate the delda so it is VA -> PA */ 130 neg x29, x29 131 132 str x1, [x0] /* modulep */ 133 str x26, [x0, 8] /* kern_l1pt */ 134 str x29, [x0, 16] /* kern_delta */ 135 adr x25, initstack 136 str x25, [x0, 24] /* kern_stack */ 137 str x24, [x0, 32] /* kern_l0pt */ 138 139 /* trace back starts here */ 140 mov fp, #0 141 /* Branch to C code */ 142 bl initarm 143 bl mi_startup 144 145 /* We should not get here */ 146 brk 0 147 148 .align 3 149.Lvirtdone: 150 .quad virtdone 151.Lbss: 152 .quad __bss_start 153.Lend: 154 .quad _end 155 156#ifdef SMP 157/* 158 * mpentry(unsigned long) 159 * 160 * Called by a core when it is being brought online. 161 * The data in x0 is passed straight to init_secondary. 162 */ 163ENTRY(mpentry) 164 /* Disable interrupts */ 165 msr daifset, #2 166 167 /* Drop to EL1 */ 168 bl drop_to_el1 169 170 /* Set the context id */ 171 msr contextidr_el1, xzr 172 173 /* Load the kernel page table */ 174 adr x24, pagetable_l0_ttbr1 175 /* Load the identity page table */ 176 adr x27, pagetable_l0_ttbr0 177 178 /* Enable the mmu */ 179 bl start_mmu 180 181 /* Jump to the virtual address space */ 182 ldr x15, =mp_virtdone 183 br x15 184 185mp_virtdone: 186 ldr x4, =secondary_stacks 187 mov x5, #(PAGE_SIZE * KSTACK_PAGES) 188 mul x5, x0, x5 189 add sp, x4, x5 190 191 b init_secondary 192END(mpentry) 193#endif 194 195/* 196 * If we are started in EL2, configure the required hypervisor 197 * registers and drop to EL1. 198 */ 199drop_to_el1: 200 mrs x1, CurrentEL 201 lsr x1, x1, #2 202 cmp x1, #0x2 203 b.eq 1f 204 ret 2051: 206 /* Configure the Hypervisor */ 207 mov x2, #(HCR_RW) 208 msr hcr_el2, x2 209 210 /* Load the Virtualization Process ID Register */ 211 mrs x2, midr_el1 212 msr vpidr_el2, x2 213 214 /* Load the Virtualization Multiprocess ID Register */ 215 mrs x2, mpidr_el1 216 msr vmpidr_el2, x2 217 218 /* Set the bits that need to be 1 in sctlr_el1 */ 219 ldr x2, .Lsctlr_res1 220 msr sctlr_el1, x2 221 222 /* Don't trap to EL2 for exceptions */ 223 mov x2, #CPTR_RES1 224 msr cptr_el2, x2 225 226 /* Don't trap to EL2 for CP15 traps */ 227 msr hstr_el2, xzr 228 229 /* Enable access to the physical timers at EL1 */ 230 mrs x2, cnthctl_el2 231 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 232 msr cnthctl_el2, x2 233 234 /* Set the counter offset to a known value */ 235 msr cntvoff_el2, xzr 236 237 /* Hypervisor trap functions */ 238 adr x2, hyp_vectors 239 msr vbar_el2, x2 240 241 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 242 msr spsr_el2, x2 243 244 /* Configure GICv3 CPU interface */ 245 mrs x2, id_aa64pfr0_el1 246 /* Extract GIC bits from the register */ 247 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 248 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 249 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 250 b.ne 2f 251 252 mrs x2, icc_sre_el2 253 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 254 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 255 msr icc_sre_el2, x2 2562: 257 258 /* Set the address to return to our return address */ 259 msr elr_el2, x30 260 isb 261 262 eret 263 264 .align 3 265.Lsctlr_res1: 266 .quad SCTLR_RES1 267 268#define VECT_EMPTY \ 269 .align 7; \ 270 1: b 1b 271 272 .align 11 273hyp_vectors: 274 VECT_EMPTY /* Synchronous EL2t */ 275 VECT_EMPTY /* IRQ EL2t */ 276 VECT_EMPTY /* FIQ EL2t */ 277 VECT_EMPTY /* Error EL2t */ 278 279 VECT_EMPTY /* Synchronous EL2h */ 280 VECT_EMPTY /* IRQ EL2h */ 281 VECT_EMPTY /* FIQ EL2h */ 282 VECT_EMPTY /* Error EL2h */ 283 284 VECT_EMPTY /* Synchronous 64-bit EL1 */ 285 VECT_EMPTY /* IRQ 64-bit EL1 */ 286 VECT_EMPTY /* FIQ 64-bit EL1 */ 287 VECT_EMPTY /* Error 64-bit EL1 */ 288 289 VECT_EMPTY /* Synchronous 32-bit EL1 */ 290 VECT_EMPTY /* IRQ 32-bit EL1 */ 291 VECT_EMPTY /* FIQ 32-bit EL1 */ 292 VECT_EMPTY /* Error 32-bit EL1 */ 293 294/* 295 * Get the delta between the physical address we were loaded to and the 296 * virtual address we expect to run from. This is used when building the 297 * initial page table. 298 */ 299get_virt_delta: 300 /* Load the physical address of virt_map */ 301 adr x29, virt_map 302 /* Load the virtual address of virt_map stored in virt_map */ 303 ldr x28, [x29] 304 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 305 sub x29, x29, x28 306 /* Find the load address for the kernel */ 307 mov x28, #(KERNBASE) 308 add x28, x28, x29 309 ret 310 311 .align 3 312virt_map: 313 .quad virt_map 314 315/* 316 * This builds the page tables containing the identity map, and the kernel 317 * virtual map. 318 * 319 * It relys on: 320 * We were loaded to an address that is on a 2MiB boundary 321 * All the memory must not cross a 1GiB boundaty 322 * x28 contains the physical address we were loaded from 323 * 324 * TODO: This is out of date. 325 * There are at least 5 pages before that address for the page tables 326 * The pages used are: 327 * - The Kernel L2 table 328 * - The Kernel L1 table 329 * - The Kernel L0 table (TTBR1) 330 * - The identity (PA = VA) L1 table 331 * - The identity (PA = VA) L0 table (TTBR0) 332 * - The DMAP L1 tables 333 */ 334create_pagetables: 335 /* Save the Link register */ 336 mov x5, x30 337 338 /* Clean the page table */ 339 adr x6, pagetable 340 mov x26, x6 341 adr x27, pagetable_end 3421: 343 stp xzr, xzr, [x6], #16 344 stp xzr, xzr, [x6], #16 345 stp xzr, xzr, [x6], #16 346 stp xzr, xzr, [x6], #16 347 cmp x6, x27 348 b.lo 1b 349 350 /* 351 * Build the TTBR1 maps. 352 */ 353 354 /* Find the size of the kernel */ 355 mov x6, #(KERNBASE) 356 /* Find modulep - begin */ 357 sub x8, x0, x6 358 /* Add two 2MiB pages for the module data and round up */ 359 ldr x7, =(3 * L2_SIZE - 1) 360 add x8, x8, x7 361 /* Get the number of l2 pages to allocate, rounded down */ 362 lsr x10, x8, #(L2_SHIFT) 363 364 /* Create the kernel space L2 table */ 365 mov x6, x26 366 mov x7, #NORMAL_MEM 367 mov x8, #(KERNBASE & L2_BLOCK_MASK) 368 mov x9, x28 369 bl build_l2_block_pagetable 370 371 /* Move to the l1 table */ 372 add x26, x26, #PAGE_SIZE 373 374 /* Link the l1 -> l2 table */ 375 mov x9, x6 376 mov x6, x26 377 bl link_l1_pagetable 378 379 /* Move to the l0 table */ 380 add x24, x26, #PAGE_SIZE 381 382 /* Link the l0 -> l1 table */ 383 mov x9, x6 384 mov x6, x24 385 mov x10, #1 386 bl link_l0_pagetable 387 388 /* Link the DMAP tables */ 389 ldr x8, =DMAP_MIN_ADDRESS 390 adr x9, pagetable_dmap; 391 mov x10, #DMAP_TABLES 392 bl link_l0_pagetable 393 394 /* 395 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_nG. 396 * They are only needed early on, so the VA = PA map is uncached. 397 */ 398 add x27, x24, #PAGE_SIZE 399 400 mov x6, x27 /* The initial page table */ 401#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 402 /* Create a table for the UART */ 403 mov x7, #(ATTR_nG | ATTR_IDX(DEVICE_MEM)) 404 mov x8, #(SOCDEV_VA) /* VA start */ 405 mov x9, #(SOCDEV_PA) /* PA start */ 406 mov x10, #1 407 bl build_l1_block_pagetable 408#endif 409 410 /* Create the VA = PA map */ 411 mov x7, #(ATTR_nG | ATTR_IDX(NORMAL_UNCACHED)) 412 mov x9, x27 413 mov x8, x9 /* VA start (== PA start) */ 414 mov x10, #1 415 bl build_l1_block_pagetable 416 417 /* Move to the l0 table */ 418 add x27, x27, #PAGE_SIZE 419 420 /* Link the l0 -> l1 table */ 421 mov x9, x6 422 mov x6, x27 423 mov x10, #1 424 bl link_l0_pagetable 425 426 /* Restore the Link register */ 427 mov x30, x5 428 ret 429 430/* 431 * Builds an L0 -> L1 table descriptor 432 * 433 * This is a link for a 512GiB block of memory with up to 1GiB regions mapped 434 * within it by build_l1_block_pagetable. 435 * 436 * x6 = L0 table 437 * x8 = Virtual Address 438 * x9 = L1 PA (trashed) 439 * x10 = Entry count 440 * x11, x12 and x13 are trashed 441 */ 442link_l0_pagetable: 443 /* 444 * Link an L0 -> L1 table entry. 445 */ 446 /* Find the table index */ 447 lsr x11, x8, #L0_SHIFT 448 and x11, x11, #L0_ADDR_MASK 449 450 /* Build the L0 block entry */ 451 mov x12, #L0_TABLE 452 453 /* Only use the output address bits */ 454 lsr x9, x9, #PAGE_SHIFT 4551: orr x13, x12, x9, lsl #PAGE_SHIFT 456 457 /* Store the entry */ 458 str x13, [x6, x11, lsl #3] 459 460 sub x10, x10, #1 461 add x11, x11, #1 462 add x9, x9, #1 463 cbnz x10, 1b 464 465 ret 466 467/* 468 * Builds an L1 -> L2 table descriptor 469 * 470 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 471 * within it by build_l2_block_pagetable. 472 * 473 * x6 = L1 table 474 * x8 = Virtual Address 475 * x9 = L2 PA (trashed) 476 * x11, x12 and x13 are trashed 477 */ 478link_l1_pagetable: 479 /* 480 * Link an L1 -> L2 table entry. 481 */ 482 /* Find the table index */ 483 lsr x11, x8, #L1_SHIFT 484 and x11, x11, #Ln_ADDR_MASK 485 486 /* Build the L1 block entry */ 487 mov x12, #L1_TABLE 488 489 /* Only use the output address bits */ 490 lsr x9, x9, #PAGE_SHIFT 491 orr x13, x12, x9, lsl #PAGE_SHIFT 492 493 /* Store the entry */ 494 str x13, [x6, x11, lsl #3] 495 496 ret 497 498/* 499 * Builds count 1 GiB page table entry 500 * x6 = L1 table 501 * x7 = Variable lower block attributes 502 * x8 = VA start 503 * x9 = PA start (trashed) 504 * x10 = Entry count 505 * x11, x12 and x13 are trashed 506 */ 507build_l1_block_pagetable: 508 /* 509 * Build the L1 table entry. 510 */ 511 /* Find the table index */ 512 lsr x11, x8, #L1_SHIFT 513 and x11, x11, #Ln_ADDR_MASK 514 515 /* Build the L1 block entry */ 516 orr x12, x7, #L1_BLOCK 517 orr x12, x12, #(ATTR_AF) 518#ifdef SMP 519 orr x12, x12, ATTR_SH(ATTR_SH_IS) 520#endif 521 522 /* Only use the output address bits */ 523 lsr x9, x9, #L1_SHIFT 524 525 /* Set the physical address for this virtual address */ 5261: orr x13, x12, x9, lsl #L1_SHIFT 527 528 /* Store the entry */ 529 str x13, [x6, x11, lsl #3] 530 531 sub x10, x10, #1 532 add x11, x11, #1 533 add x9, x9, #1 534 cbnz x10, 1b 535 536 ret 537 538/* 539 * Builds count 2 MiB page table entry 540 * x6 = L2 table 541 * x7 = Type (0 = Device, 1 = Normal) 542 * x8 = VA start 543 * x9 = PA start (trashed) 544 * x10 = Entry count 545 * x11, x12 and x13 are trashed 546 */ 547build_l2_block_pagetable: 548 /* 549 * Build the L2 table entry. 550 */ 551 /* Find the table index */ 552 lsr x11, x8, #L2_SHIFT 553 and x11, x11, #Ln_ADDR_MASK 554 555 /* Build the L2 block entry */ 556 lsl x12, x7, #2 557 orr x12, x12, #L2_BLOCK 558 orr x12, x12, #(ATTR_AF) 559 orr x12, x12, #(ATTR_UXN) 560#ifdef SMP 561 orr x12, x12, ATTR_SH(ATTR_SH_IS) 562#endif 563 564 /* Only use the output address bits */ 565 lsr x9, x9, #L2_SHIFT 566 567 /* Set the physical address for this virtual address */ 5681: orr x13, x12, x9, lsl #L2_SHIFT 569 570 /* Store the entry */ 571 str x13, [x6, x11, lsl #3] 572 573 sub x10, x10, #1 574 add x11, x11, #1 575 add x9, x9, #1 576 cbnz x10, 1b 577 578 ret 579 580start_mmu: 581 dsb sy 582 583 /* Load the exception vectors */ 584 ldr x2, =exception_vectors 585 msr vbar_el1, x2 586 587 /* Load ttbr0 and ttbr1 */ 588 msr ttbr0_el1, x27 589 msr ttbr1_el1, x24 590 isb 591 592 /* Clear the Monitor Debug System control register */ 593 msr mdscr_el1, xzr 594 595 /* Invalidate the TLB */ 596 tlbi vmalle1is 597 598 ldr x2, mair 599 msr mair_el1, x2 600 601 /* 602 * Setup TCR according to the PARange and ASIDBits fields 603 * from ID_AA64MMFR0_EL1. More precisely, set TCR_EL1.AS 604 * to 1 only if the ASIDBits field equals 0b0010. 605 */ 606 ldr x2, tcr 607 mrs x3, id_aa64mmfr0_el1 608 bfi x2, x3, #32, #3 609 and x3, x3, #0xF0 610 cmp x3, #0x20 611 cset x3, eq 612 bfi x2, x3, #36, #1 613 msr tcr_el1, x2 614 615 /* Setup SCTLR */ 616 ldr x2, sctlr_set 617 ldr x3, sctlr_clear 618 mrs x1, sctlr_el1 619 bic x1, x1, x3 /* Clear the required bits */ 620 orr x1, x1, x2 /* Set the required bits */ 621 msr sctlr_el1, x1 622 isb 623 624 ret 625 626 .align 3 627mair: 628 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, 0) | \ 629 MAIR_ATTR(MAIR_NORMAL_NC, 1) | \ 630 MAIR_ATTR(MAIR_NORMAL_WB, 2) | \ 631 MAIR_ATTR(MAIR_NORMAL_WT, 3) 632tcr: 633 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG1_4K | \ 634 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 635sctlr_set: 636 /* Bits to set */ 637 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 638 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 639 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 640 SCTLR_M | SCTLR_CP15BEN) 641sctlr_clear: 642 /* Bits to clear */ 643 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 644 SCTLR_ITD | SCTLR_A) 645 646 .globl abort 647abort: 648 b abort 649 650 //.section .init_pagetable 651 .align 12 /* 4KiB aligned */ 652 /* 653 * 3 initial tables (in the following order): 654 * L2 for kernel (High addresses) 655 * L1 for kernel 656 * L1 for user (Low addresses) 657 */ 658pagetable: 659 .space PAGE_SIZE 660pagetable_l1_ttbr1: 661 .space PAGE_SIZE 662pagetable_l0_ttbr1: 663 .space PAGE_SIZE 664pagetable_l1_ttbr0: 665 .space PAGE_SIZE 666pagetable_l0_ttbr0: 667 .space PAGE_SIZE 668 669 .globl pagetable_dmap 670pagetable_dmap: 671 .space PAGE_SIZE * DMAP_TABLES 672pagetable_end: 673 674el2_pagetable: 675 .space PAGE_SIZE 676 677 .globl init_pt_va 678init_pt_va: 679 .quad pagetable /* XXX: Keep page tables VA */ 680 681 .align 4 682initstack: 683 .space (PAGE_SIZE * KSTACK_PAGES) 684initstack_end: 685 686 687ENTRY(sigcode) 688 mov x0, sp 689 add x0, x0, #SF_UC 690 6911: 692 mov x8, #SYS_sigreturn 693 svc 0 694 695 /* sigreturn failed, exit */ 696 mov x8, #SYS_exit 697 svc 0 698 699 b 1b 700END(sigcode) 701 /* This may be copied to the stack, keep it 16-byte aligned */ 702 .align 3 703esigcode: 704 705 .data 706 .align 3 707 .global szsigcode 708szsigcode: 709 .quad esigcode - sigcode 710 711ENTRY(aarch32_sigcode) 712 .word 0xe1a0000d // mov r0, sp 713 .word 0xe2800040 // add r0, r0, #SIGF_UC 714 .word 0xe59f700c // ldr r7, [pc, #12] 715 .word 0xef000000 // swi #0 716 .word 0xe59f7008 // ldr r7, [pc, #8] 717 .word 0xef000000 // swi #0 718 .word 0xeafffffa // b . - 16 719END(aarch32_sigcode) 720 .word SYS_sigreturn 721 .word SYS_exit 722 .align 3 723aarch32_esigcode: 724 .data 725 .global sz_aarch32_sigcode 726sz_aarch32_sigcode: 727 .quad aarch32_esigcode - aarch32_sigcode 728