1/* 2 * Low-level CPU initialisation 3 * Based on arch/arm/kernel/head.S 4 * 5 * Copyright (C) 1994-2002 Russell King 6 * Copyright (C) 2003-2012 ARM Ltd. 7 * Authors: Catalin Marinas <catalin.marinas@arm.com> 8 * Will Deacon <will.deacon@arm.com> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program. If not, see <http://www.gnu.org/licenses/>. 21 */ 22 23#include <linux/linkage.h> 24#include <linux/init.h> 25 26#include <asm/assembler.h> 27#include <asm/ptrace.h> 28#include <asm/asm-offsets.h> 29#include <asm/memory.h> 30#include <asm/thread_info.h> 31#include <asm/pgtable-hwdef.h> 32#include <asm/pgtable.h> 33#include <asm/page.h> 34 35/* 36 * swapper_pg_dir is the virtual address of the initial page table. We place 37 * the page tables 3 * PAGE_SIZE below KERNEL_RAM_VADDR. The idmap_pg_dir has 38 * 2 pages and is placed below swapper_pg_dir. 39 */ 40#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) 41 42#if (KERNEL_RAM_VADDR & 0xfffff) != 0x80000 43#error KERNEL_RAM_VADDR must start at 0xXXX80000 44#endif 45 46#define SWAPPER_DIR_SIZE (3 * PAGE_SIZE) 47#define IDMAP_DIR_SIZE (2 * PAGE_SIZE) 48 49 .globl swapper_pg_dir 50 .equ swapper_pg_dir, KERNEL_RAM_VADDR - SWAPPER_DIR_SIZE 51 52 .globl idmap_pg_dir 53 .equ idmap_pg_dir, swapper_pg_dir - IDMAP_DIR_SIZE 54 55 .macro pgtbl, ttb0, ttb1, phys 56 add \ttb1, \phys, #TEXT_OFFSET - SWAPPER_DIR_SIZE 57 sub \ttb0, \ttb1, #IDMAP_DIR_SIZE 58 .endm 59 60#ifdef CONFIG_ARM64_64K_PAGES 61#define BLOCK_SHIFT PAGE_SHIFT 62#define BLOCK_SIZE PAGE_SIZE 63#else 64#define BLOCK_SHIFT SECTION_SHIFT 65#define BLOCK_SIZE SECTION_SIZE 66#endif 67 68#define KERNEL_START KERNEL_RAM_VADDR 69#define KERNEL_END _end 70 71/* 72 * Initial memory map attributes. 73 */ 74#ifndef CONFIG_SMP 75#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF 76#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF 77#else 78#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED 79#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S 80#endif 81 82#ifdef CONFIG_ARM64_64K_PAGES 83#define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS 84#define IO_MMUFLAGS PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_XN | PTE_FLAGS 85#else 86#define MM_MMUFLAGS PMD_ATTRINDX(MT_NORMAL) | PMD_FLAGS 87#define IO_MMUFLAGS PMD_ATTRINDX(MT_DEVICE_nGnRE) | PMD_SECT_XN | PMD_FLAGS 88#endif 89 90/* 91 * Kernel startup entry point. 92 * --------------------------- 93 * 94 * The requirements are: 95 * MMU = off, D-cache = off, I-cache = on or off, 96 * x0 = physical address to the FDT blob. 97 * 98 * This code is mostly position independent so you call this at 99 * __pa(PAGE_OFFSET + TEXT_OFFSET). 100 * 101 * Note that the callee-saved registers are used for storing variables 102 * that are useful before the MMU is enabled. The allocations are described 103 * in the entry routines. 104 */ 105 __HEAD 106 107 /* 108 * DO NOT MODIFY. Image header expected by Linux boot-loaders. 109 */ 110 b stext // branch to kernel start, magic 111 .long 0 // reserved 112 .quad TEXT_OFFSET // Image load offset from start of RAM 113 .quad 0 // reserved 114 .quad 0 // reserved 115 116ENTRY(stext) 117 mov x21, x0 // x21=FDT 118 bl el2_setup // Drop to EL1 119 mrs x22, midr_el1 // x22=cpuid 120 mov x0, x22 121 bl lookup_processor_type 122 mov x23, x0 // x23=current cpu_table 123 cbz x23, __error_p // invalid processor (x23=0)? 124 bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET 125 bl __vet_fdt 126 bl __create_page_tables // x25=TTBR0, x26=TTBR1 127 /* 128 * The following calls CPU specific code in a position independent 129 * manner. See arch/arm64/mm/proc.S for details. x23 = base of 130 * cpu_info structure selected by lookup_processor_type above. 131 * On return, the CPU will be ready for the MMU to be turned on and 132 * the TCR will have been set. 133 */ 134 ldr x27, __switch_data // address to jump to after 135 // MMU has been enabled 136 adr lr, __enable_mmu // return (PIC) address 137 ldr x12, [x23, #CPU_INFO_SETUP] 138 add x12, x12, x28 // __virt_to_phys 139 br x12 // initialise processor 140ENDPROC(stext) 141 142/* 143 * If we're fortunate enough to boot at EL2, ensure that the world is 144 * sane before dropping to EL1. 145 */ 146ENTRY(el2_setup) 147 mrs x0, CurrentEL 148 cmp x0, #PSR_MODE_EL2t 149 ccmp x0, #PSR_MODE_EL2h, #0x4, ne 150 b.eq 1f 151 ret 152 153 /* Hyp configuration. */ 1541: mov x0, #(1 << 31) // 64-bit EL1 155 msr hcr_el2, x0 156 157 /* Generic timers. */ 158 mrs x0, cnthctl_el2 159 orr x0, x0, #3 // Enable EL1 physical timers 160 msr cnthctl_el2, x0 161 162 /* Populate ID registers. */ 163 mrs x0, midr_el1 164 mrs x1, mpidr_el1 165 msr vpidr_el2, x0 166 msr vmpidr_el2, x1 167 168 /* sctlr_el1 */ 169 mov x0, #0x0800 // Set/clear RES{1,0} bits 170 movk x0, #0x30d0, lsl #16 171 msr sctlr_el1, x0 172 173 /* Coprocessor traps. */ 174 mov x0, #0x33ff 175 msr cptr_el2, x0 // Disable copro. traps to EL2 176 177#ifdef CONFIG_COMPAT 178 msr hstr_el2, xzr // Disable CP15 traps to EL2 179#endif 180 181 /* spsr */ 182 mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ 183 PSR_MODE_EL1h) 184 msr spsr_el2, x0 185 msr elr_el2, lr 186 eret 187ENDPROC(el2_setup) 188 189 .align 3 1902: .quad . 191 .quad PAGE_OFFSET 192 193#ifdef CONFIG_SMP 194 .pushsection .smp.pen.text, "ax" 195 .align 3 1961: .quad . 197 .quad secondary_holding_pen_release 198 199 /* 200 * This provides a "holding pen" for platforms to hold all secondary 201 * cores are held until we're ready for them to initialise. 202 */ 203ENTRY(secondary_holding_pen) 204 bl el2_setup // Drop to EL1 205 mrs x0, mpidr_el1 206 and x0, x0, #15 // CPU number 207 adr x1, 1b 208 ldp x2, x3, [x1] 209 sub x1, x1, x2 210 add x3, x3, x1 211pen: ldr x4, [x3] 212 cmp x4, x0 213 b.eq secondary_startup 214 wfe 215 b pen 216ENDPROC(secondary_holding_pen) 217 .popsection 218 219ENTRY(secondary_startup) 220 /* 221 * Common entry point for secondary CPUs. 222 */ 223 mrs x22, midr_el1 // x22=cpuid 224 mov x0, x22 225 bl lookup_processor_type 226 mov x23, x0 // x23=current cpu_table 227 cbz x23, __error_p // invalid processor (x23=0)? 228 229 bl __calc_phys_offset // x24=phys offset 230 pgtbl x25, x26, x24 // x25=TTBR0, x26=TTBR1 231 ldr x12, [x23, #CPU_INFO_SETUP] 232 add x12, x12, x28 // __virt_to_phys 233 blr x12 // initialise processor 234 235 ldr x21, =secondary_data 236 ldr x27, =__secondary_switched // address to jump to after enabling the MMU 237 b __enable_mmu 238ENDPROC(secondary_startup) 239 240ENTRY(__secondary_switched) 241 ldr x0, [x21] // get secondary_data.stack 242 mov sp, x0 243 mov x29, #0 244 b secondary_start_kernel 245ENDPROC(__secondary_switched) 246#endif /* CONFIG_SMP */ 247 248/* 249 * Setup common bits before finally enabling the MMU. Essentially this is just 250 * loading the page table pointer and vector base registers. 251 * 252 * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on 253 * the MMU. 254 */ 255__enable_mmu: 256 ldr x5, =vectors 257 msr vbar_el1, x5 258 msr ttbr0_el1, x25 // load TTBR0 259 msr ttbr1_el1, x26 // load TTBR1 260 isb 261 b __turn_mmu_on 262ENDPROC(__enable_mmu) 263 264/* 265 * Enable the MMU. This completely changes the structure of the visible memory 266 * space. You will not be able to trace execution through this. 267 * 268 * x0 = system control register 269 * x27 = *virtual* address to jump to upon completion 270 * 271 * other registers depend on the function called upon completion 272 */ 273 .align 6 274__turn_mmu_on: 275 msr sctlr_el1, x0 276 isb 277 br x27 278ENDPROC(__turn_mmu_on) 279 280/* 281 * Calculate the start of physical memory. 282 */ 283__calc_phys_offset: 284 adr x0, 1f 285 ldp x1, x2, [x0] 286 sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET 287 add x24, x2, x28 // x24 = PHYS_OFFSET 288 ret 289ENDPROC(__calc_phys_offset) 290 291 .align 3 2921: .quad . 293 .quad PAGE_OFFSET 294 295/* 296 * Macro to populate the PGD for the corresponding block entry in the next 297 * level (tbl) for the given virtual address. 298 * 299 * Preserves: pgd, tbl, virt 300 * Corrupts: tmp1, tmp2 301 */ 302 .macro create_pgd_entry, pgd, tbl, virt, tmp1, tmp2 303 lsr \tmp1, \virt, #PGDIR_SHIFT 304 and \tmp1, \tmp1, #PTRS_PER_PGD - 1 // PGD index 305 orr \tmp2, \tbl, #3 // PGD entry table type 306 str \tmp2, [\pgd, \tmp1, lsl #3] 307 .endm 308 309/* 310 * Macro to populate block entries in the page table for the start..end 311 * virtual range (inclusive). 312 * 313 * Preserves: tbl, flags 314 * Corrupts: phys, start, end, pstate 315 */ 316 .macro create_block_map, tbl, flags, phys, start, end, idmap=0 317 lsr \phys, \phys, #BLOCK_SHIFT 318 .if \idmap 319 and \start, \phys, #PTRS_PER_PTE - 1 // table index 320 .else 321 lsr \start, \start, #BLOCK_SHIFT 322 and \start, \start, #PTRS_PER_PTE - 1 // table index 323 .endif 324 orr \phys, \flags, \phys, lsl #BLOCK_SHIFT // table entry 325 .ifnc \start,\end 326 lsr \end, \end, #BLOCK_SHIFT 327 and \end, \end, #PTRS_PER_PTE - 1 // table end index 328 .endif 3299999: str \phys, [\tbl, \start, lsl #3] // store the entry 330 .ifnc \start,\end 331 add \start, \start, #1 // next entry 332 add \phys, \phys, #BLOCK_SIZE // next block 333 cmp \start, \end 334 b.ls 9999b 335 .endif 336 .endm 337 338/* 339 * Setup the initial page tables. We only setup the barest amount which is 340 * required to get the kernel running. The following sections are required: 341 * - identity mapping to enable the MMU (low address, TTBR0) 342 * - first few MB of the kernel linear mapping to jump to once the MMU has 343 * been enabled, including the FDT blob (TTBR1) 344 */ 345__create_page_tables: 346 pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses 347 348 /* 349 * Clear the idmap and swapper page tables. 350 */ 351 mov x0, x25 352 add x6, x26, #SWAPPER_DIR_SIZE 3531: stp xzr, xzr, [x0], #16 354 stp xzr, xzr, [x0], #16 355 stp xzr, xzr, [x0], #16 356 stp xzr, xzr, [x0], #16 357 cmp x0, x6 358 b.lo 1b 359 360 ldr x7, =MM_MMUFLAGS 361 362 /* 363 * Create the identity mapping. 364 */ 365 add x0, x25, #PAGE_SIZE // section table address 366 adr x3, __turn_mmu_on // virtual/physical address 367 create_pgd_entry x25, x0, x3, x5, x6 368 create_block_map x0, x7, x3, x5, x5, idmap=1 369 370 /* 371 * Map the kernel image (starting with PHYS_OFFSET). 372 */ 373 add x0, x26, #PAGE_SIZE // section table address 374 mov x5, #PAGE_OFFSET 375 create_pgd_entry x26, x0, x5, x3, x6 376 ldr x6, =KERNEL_END - 1 377 mov x3, x24 // phys offset 378 create_block_map x0, x7, x3, x5, x6 379 380 /* 381 * Map the FDT blob (maximum 2MB; must be within 512MB of 382 * PHYS_OFFSET). 383 */ 384 mov x3, x21 // FDT phys address 385 and x3, x3, #~((1 << 21) - 1) // 2MB aligned 386 mov x6, #PAGE_OFFSET 387 sub x5, x3, x24 // subtract PHYS_OFFSET 388 tst x5, #~((1 << 29) - 1) // within 512MB? 389 csel x21, xzr, x21, ne // zero the FDT pointer 390 b.ne 1f 391 add x5, x5, x6 // __va(FDT blob) 392 add x6, x5, #1 << 21 // 2MB for the FDT blob 393 sub x6, x6, #1 // inclusive range 394 create_block_map x0, x7, x3, x5, x6 3951: 396 ret 397ENDPROC(__create_page_tables) 398 .ltorg 399 400 .align 3 401 .type __switch_data, %object 402__switch_data: 403 .quad __mmap_switched 404 .quad __data_loc // x4 405 .quad _data // x5 406 .quad __bss_start // x6 407 .quad _end // x7 408 .quad processor_id // x4 409 .quad __fdt_pointer // x5 410 .quad memstart_addr // x6 411 .quad init_thread_union + THREAD_START_SP // sp 412 413/* 414 * The following fragment of code is executed with the MMU on in MMU mode, and 415 * uses absolute addresses; this is not position independent. 416 */ 417__mmap_switched: 418 adr x3, __switch_data + 8 419 420 ldp x4, x5, [x3], #16 421 ldp x6, x7, [x3], #16 422 cmp x4, x5 // Copy data segment if needed 4231: ccmp x5, x6, #4, ne 424 b.eq 2f 425 ldr x16, [x4], #8 426 str x16, [x5], #8 427 b 1b 4282: 4291: cmp x6, x7 430 b.hs 2f 431 str xzr, [x6], #8 // Clear BSS 432 b 1b 4332: 434 ldp x4, x5, [x3], #16 435 ldr x6, [x3], #8 436 ldr x16, [x3] 437 mov sp, x16 438 str x22, [x4] // Save processor ID 439 str x21, [x5] // Save FDT pointer 440 str x24, [x6] // Save PHYS_OFFSET 441 mov x29, #0 442 b start_kernel 443ENDPROC(__mmap_switched) 444 445/* 446 * Exception handling. Something went wrong and we can't proceed. We ought to 447 * tell the user, but since we don't have any guarantee that we're even 448 * running on the right architecture, we do virtually nothing. 449 */ 450__error_p: 451ENDPROC(__error_p) 452 453__error: 4541: nop 455 b 1b 456ENDPROC(__error) 457 458/* 459 * This function gets the processor ID in w0 and searches the cpu_table[] for 460 * a match. It returns a pointer to the struct cpu_info it found. The 461 * cpu_table[] must end with an empty (all zeros) structure. 462 * 463 * This routine can be called via C code and it needs to work with the MMU 464 * both disabled and enabled (the offset is calculated automatically). 465 */ 466ENTRY(lookup_processor_type) 467 adr x1, __lookup_processor_type_data 468 ldp x2, x3, [x1] 469 sub x1, x1, x2 // get offset between VA and PA 470 add x3, x3, x1 // convert VA to PA 4711: 472 ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask 473 cbz w5, 2f // end of list? 474 and w6, w6, w0 475 cmp w5, w6 476 b.eq 3f 477 add x3, x3, #CPU_INFO_SZ 478 b 1b 4792: 480 mov x3, #0 // unknown processor 4813: 482 mov x0, x3 483 ret 484ENDPROC(lookup_processor_type) 485 486 .align 3 487 .type __lookup_processor_type_data, %object 488__lookup_processor_type_data: 489 .quad . 490 .quad cpu_table 491 .size __lookup_processor_type_data, . - __lookup_processor_type_data 492 493/* 494 * Determine validity of the x21 FDT pointer. 495 * The dtb must be 8-byte aligned and live in the first 512M of memory. 496 */ 497__vet_fdt: 498 tst x21, #0x7 499 b.ne 1f 500 cmp x21, x24 501 b.lt 1f 502 mov x0, #(1 << 29) 503 add x0, x0, x24 504 cmp x21, x0 505 b.ge 1f 506 ret 5071: 508 mov x21, #0 509 ret 510ENDPROC(__vet_fdt) 511