1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.s" 30#include <sys/syscall.h> 31#include <machine/asm.h> 32#include <machine/armreg.h> 33#include <machine/hypervisor.h> 34#include <machine/param.h> 35#include <machine/pte.h> 36 37#define VIRT_BITS 39 38 39 .globl kernbase 40 .set kernbase, KERNBASE 41 42#define DEVICE_MEM 0 43#define NORMAL_UNCACHED 1 44#define NORMAL_MEM 2 45 46/* 47 * We assume: 48 * MMU on with an identity map, or off 49 * D-Cache: off 50 * I-Cache: on or off 51 * We are loaded at a 2MiB aligned address 52 */ 53 54#define INIT_STACK_SIZE (PAGE_SIZE * 4) 55 56 .text 57 .globl _start 58_start: 59 /* Drop to EL1 */ 60 bl drop_to_el1 61 62 /* 63 * Disable the MMU. We may have entered the kernel with it on and 64 * will need to update the tables later. If this has been set up 65 * with anything other than a VA == PA map then this will fail, 66 * but in this case the code to find where we are running from 67 * would have also failed. 68 */ 69 dsb sy 70 mrs x2, sctlr_el1 71 bic x2, x2, SCTLR_M 72 msr sctlr_el1, x2 73 isb 74 75 76 /* Get the virt -> phys offset */ 77 bl get_virt_delta 78 79 /* 80 * At this point: 81 * x29 = PA - VA 82 * x28 = Our physical load address 83 */ 84 85 /* Create the page tables */ 86 bl create_pagetables 87 88 /* 89 * At this point: 90 * x27 = TTBR0 table 91 * x26 = TTBR1 table 92 */ 93 94 /* Enable the mmu */ 95 bl start_mmu 96 97 /* Jump to the virtual address space */ 98 ldr x15, .Lvirtdone 99 br x15 100 101virtdone: 102 /* Set up the stack */ 103 adr x25, initstack_end 104 mov sp, x25 105 sub sp, sp, #PCB_SIZE 106 107 /* Zero the BSS */ 108 ldr x15, .Lbss 109 ldr x14, .Lend 1101: 111 str xzr, [x15], #8 112 cmp x15, x14 113 b.lo 1b 114 115 /* Backup the module pointer */ 116 mov x1, x0 117 118 /* Make the page table base a virtual address */ 119 sub x26, x26, x29 120 121 sub sp, sp, #(64 * 4) 122 mov x0, sp 123 124 /* Degate the delda so it is VA -> PA */ 125 neg x29, x29 126 127 str x1, [x0] /* modulep */ 128 str x26, [x0, 8] /* kern_l1pt */ 129 str x29, [x0, 16] /* kern_delta */ 130 str x25, [x0, 24] /* kern_stack */ 131 132 /* trace back starts here */ 133 mov fp, #0 134 /* Branch to C code */ 135 bl initarm 136 bl mi_startup 137 138 /* We should not get here */ 139 brk 0 140 141 .align 3 142.Lvirtdone: 143 .quad virtdone 144.Lbss: 145 .quad __bss_start 146.Lend: 147 .quad _end 148 149/* 150 * If we are started in EL2, configure the required hypervisor 151 * registers and drop to EL1. 152 */ 153drop_to_el1: 154 mrs x1, CurrentEL 155 lsr x1, x1, #2 156 cmp x1, #0x2 157 b.eq 1f 158 ret 1591: 160 /* Configure the Hypervisor */ 161 mov x2, #(HCR_RW) 162 msr hcr_el2, x2 163 164 /* Load the Virtualization Process ID Register */ 165 mrs x2, midr_el1 166 msr vpidr_el2, x2 167 168 /* Load the Virtualization Multiprocess ID Register */ 169 mrs x2, mpidr_el1 170 msr vmpidr_el2, x2 171 172 /* Set the bits that need to be 1 in sctlr_el1 */ 173 ldr x2, .Lsctlr_res1 174 msr sctlr_el1, x2 175 176 /* Don't trap to EL2 for exceptions */ 177 mov x2, #CPTR_RES1 178 msr cptr_el2, x2 179 180 /* Don't trap to EL2 for CP15 traps */ 181 msr hstr_el2, xzr 182 183 /* Hypervisor trap functions */ 184 adr x2, hyp_vectors 185 msr vbar_el2, x2 186 187 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 188 msr spsr_el2, x2 189 190 /* Configure GICv3 CPU interface */ 191 mrs x2, id_aa64pfr0_el1 192 /* Extract GIC bits from the register */ 193 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 194 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 195 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 196 b.ne 2f 197 198 mrs x2, icc_sre_el2 199 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 200 msr icc_sre_el2, x2 201 isb 2022: 203 204 /* Set the address to return to our return address */ 205 msr elr_el2, x30 206 207 eret 208 209 .align 3 210.Lsctlr_res1: 211 .quad SCTLR_RES1 212 213#define VECT_EMPTY \ 214 .align 7; \ 215 1: b 1b 216 217 .align 11 218hyp_vectors: 219 VECT_EMPTY /* Synchronous EL2t */ 220 VECT_EMPTY /* IRQ EL2t */ 221 VECT_EMPTY /* FIQ EL2t */ 222 VECT_EMPTY /* Error EL2t */ 223 224 VECT_EMPTY /* Synchronous EL2h */ 225 VECT_EMPTY /* IRQ EL2h */ 226 VECT_EMPTY /* FIQ EL2h */ 227 VECT_EMPTY /* Error EL2h */ 228 229 VECT_EMPTY /* Synchronous 64-bit EL1 */ 230 VECT_EMPTY /* IRQ 64-bit EL1 */ 231 VECT_EMPTY /* FIQ 64-bit EL1 */ 232 VECT_EMPTY /* Error 64-bit EL1 */ 233 234 VECT_EMPTY /* Synchronous 32-bit EL1 */ 235 VECT_EMPTY /* IRQ 32-bit EL1 */ 236 VECT_EMPTY /* FIQ 32-bit EL1 */ 237 VECT_EMPTY /* Error 32-bit EL1 */ 238 239/* 240 * Get the delta between the physical address we were loaded to and the 241 * virtual address we expect to run from. This is used when building the 242 * initial page table. 243 */ 244get_virt_delta: 245 /* Load the physical address of virt_map */ 246 adr x29, virt_map 247 /* Load the virtual address of virt_map stored in virt_map */ 248 ldr x28, [x29] 249 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 250 sub x29, x29, x28 251 /* Find the load address for the kernel */ 252 mov x28, #(KERNBASE) 253 add x28, x28, x29 254 ret 255 256 .align 3 257virt_map: 258 .quad virt_map 259 260/* 261 * This builds the page tables containing the identity map, and the kernel 262 * virtual map. 263 * 264 * It relys on: 265 * We were loaded to an address that is on a 2MiB boundary 266 * All the memory must not cross a 1GiB boundaty 267 * x28 contains the physical address we were loaded from 268 * 269 * TODO: This is out of date. 270 * There are at least 5 pages before that address for the page tables 271 * The pages used are: 272 * - The identity (PA = VA) table (TTBR0) 273 * - The Kernel L1 table (TTBR1)(not yet) 274 * - The PA != VA L2 table to jump into (not yet) 275 * - The FDT L2 table (not yet) 276 */ 277create_pagetables: 278 /* Save the Link register */ 279 mov x5, x30 280 281 /* Clean the page table */ 282 adr x6, pagetable 283 mov x26, x6 284 adr x27, pagetable_end 2851: 286 stp xzr, xzr, [x6], #16 287 stp xzr, xzr, [x6], #16 288 stp xzr, xzr, [x6], #16 289 stp xzr, xzr, [x6], #16 290 cmp x6, x27 291 b.lo 1b 292 293 /* 294 * Build the TTBR1 maps. 295 */ 296 297 /* Find the size of the kernel */ 298 mov x6, #(KERNBASE) 299 ldr x7, .Lend 300 /* Find the end - begin */ 301 sub x8, x7, x6 302 /* Get the number of l2 pages to allocate, rounded down */ 303 lsr x10, x8, #(L2_SHIFT) 304 /* Add 4 MiB for any rounding above and the module data */ 305 add x10, x10, #2 306 307 /* Create the kernel space L2 table */ 308 mov x6, x26 309 mov x7, #NORMAL_MEM 310 mov x8, #(KERNBASE & L2_BLOCK_MASK) 311 mov x9, x28 312 bl build_block_pagetable 313 314 /* Move to the l1 table */ 315 add x26, x26, #PAGE_SIZE 316 317 /* Link the l1 -> l2 table */ 318 mov x9, x6 319 mov x6, x26 320 bl link_l1_pagetable 321 322 323 /* 324 * Build the TTBR0 maps. 325 */ 326 add x27, x26, #PAGE_SIZE 327 328#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 329 /* Create a table for the UART */ 330 mov x6, x27 /* The initial page table */ 331 mov x7, #DEVICE_MEM 332 mov x8, #(SOCDEV_VA) /* VA start */ 333 mov x9, #(SOCDEV_PA) /* PA start */ 334 bl build_section_pagetable 335#endif 336 337 /* Create the VA = PA map */ 338 mov x6, x27 /* The initial page table */ 339 mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ 340 mov x9, x27 341 mov x8, x9 /* VA start (== PA start) */ 342 bl build_section_pagetable 343 344 /* Restore the Link register */ 345 mov x30, x5 346 ret 347 348/* 349 * Builds a 1 GiB page table entry 350 * x6 = L1 table 351 * x7 = Type (0 = Device, 1 = Normal) 352 * x8 = VA start 353 * x9 = PA start (trashed) 354 * x11, x12 and x13 are trashed 355 */ 356build_section_pagetable: 357 /* 358 * Build the L1 table entry. 359 */ 360 /* Find the table index */ 361 lsr x11, x8, #L1_SHIFT 362 and x11, x11, #Ln_ADDR_MASK 363 364 /* Build the L1 block entry */ 365 lsl x12, x7, #2 366 orr x12, x12, #L1_BLOCK 367 orr x12, x12, #(ATTR_AF) 368 369 /* Only use the output address bits */ 370 lsr x9, x9, #L1_SHIFT 371 orr x12, x12, x9, lsl #L1_SHIFT 372 373 /* Store the entry */ 374 str x12, [x6, x11, lsl #3] 375 376 ret 377 378/* 379 * Builds an L1 -> L2 table descriptor 380 * 381 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 382 * within it by build_block_pagetable. 383 * 384 * x6 = L1 table 385 * x8 = Virtual Address 386 * x9 = L2 PA (trashed) 387 * x11, x12 and x13 are trashed 388 */ 389link_l1_pagetable: 390 /* 391 * Link an L1 -> L2 table entry. 392 */ 393 /* Find the table index */ 394 lsr x11, x8, #L1_SHIFT 395 and x11, x11, #Ln_ADDR_MASK 396 397 /* Build the L1 block entry */ 398 mov x12, #L1_TABLE 399 400 /* Only use the output address bits */ 401 lsr x9, x9, #12 402 orr x12, x12, x9, lsl #12 403 404 /* Store the entry */ 405 str x12, [x6, x11, lsl #3] 406 407 ret 408 409/* 410 * Builds count 2 MiB page table entry 411 * x6 = L2 table 412 * x7 = Type (0 = Device, 1 = Normal) 413 * x8 = VA start 414 * x9 = PA start (trashed) 415 * x10 = Entry count (TODO) 416 * x11, x12 and x13 are trashed 417 */ 418build_block_pagetable: 419 /* 420 * Build the L2 table entry. 421 */ 422 /* Find the table index */ 423 lsr x11, x8, #L2_SHIFT 424 and x11, x11, #Ln_ADDR_MASK 425 426 /* Build the L2 block entry */ 427 lsl x12, x7, #2 428 orr x12, x12, #L2_BLOCK 429 orr x12, x12, #(ATTR_AF) 430 431 /* Only use the output address bits */ 432 lsr x9, x9, #L2_SHIFT 433 434 /* Set the physical address for this virtual address */ 4351: orr x12, x12, x9, lsl #L2_SHIFT 436 437 /* Store the entry */ 438 str x12, [x6, x11, lsl #3] 439 440 /* Clear the address bits */ 441 and x12, x12, #ATTR_MASK_L 442 443 sub x10, x10, #1 444 add x11, x11, #1 445 add x9, x9, #1 446 cbnz x10, 1b 447 4482: ret 449 450start_mmu: 451 dsb sy 452 453 /* Load the exception vectors */ 454 ldr x2, =exception_vectors 455 msr vbar_el1, x2 456 457 /* Load ttbr0 and ttbr1 */ 458 msr ttbr0_el1, x27 459 msr ttbr1_el1, x26 460 isb 461 462 /* Clear the Monitor Debug System control register */ 463 msr mdscr_el1, xzr 464 465 /* Invalidate the TLB */ 466 tlbi vmalle1is 467 468 ldr x2, mair 469 msr mair_el1, x2 470 471 /* Setup TCR according to PARange bits from ID_AA64MMFR0_EL1 */ 472 ldr x2, tcr 473 mrs x3, id_aa64mmfr0_el1 474 bfi x2, x3, #32, #3 475 msr tcr_el1, x2 476 477 /* Setup SCTLR */ 478 ldr x2, sctlr_set 479 ldr x3, sctlr_clear 480 mrs x1, sctlr_el1 481 bic x1, x1, x3 /* Clear the required bits */ 482 orr x1, x1, x2 /* Set the required bits */ 483 msr sctlr_el1, x1 484 isb 485 486 ret 487 488 .align 3 489mair: 490 /* Device Normal, no cache Normal, write-back */ 491 .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) 492tcr: 493 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K) 494sctlr_set: 495 /* Bits to set */ 496 .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 497 SCTLR_I | SCTLR_SED | SCTLR_C | SCTLR_M) 498sctlr_clear: 499 /* Bits to clear */ 500 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ 501 SCTLR_THEE | SCTLR_CP15BEN | SCTLR_SA0 | SCTLR_SA | SCTLR_A) 502 503 .globl abort 504abort: 505 b abort 506 507 //.section .init_pagetable 508 .align 12 /* 4KiB aligned */ 509 /* 510 * 3 initial tables (in the following order): 511 * L2 for kernel (High addresses) 512 * L1 for kernel 513 * L1 for user (Low addresses) 514 */ 515pagetable: 516 .space PAGE_SIZE 517pagetable_l1_ttbr1: 518 .space PAGE_SIZE 519pagetable_l1_ttbr0: 520 .space PAGE_SIZE 521pagetable_end: 522 523el2_pagetable: 524 .space PAGE_SIZE 525 526 .globl init_pt_va 527init_pt_va: 528 .quad pagetable /* XXX: Keep page tables VA */ 529 530 .align 4 531initstack: 532 .space (PAGE_SIZE * KSTACK_PAGES) 533initstack_end: 534 535 536ENTRY(sigcode) 537 mov x0, sp 538 add x0, x0, #SF_UC 539 5401: 541 mov x8, #SYS_sigreturn 542 svc 0 543 544 /* sigreturn failed, exit */ 545 mov x8, #SYS_exit 546 svc 0 547 548 b 1b 549END(sigcode) 550 /* This may be copied to the stack, keep it 16-byte aligned */ 551 .align 3 552esigcode: 553 554 .data 555 .align 3 556 .global szsigcode 557szsigcode: 558 .quad esigcode - sigcode 559