head.S (e6128a8e523cfa8f46a1501e6432c37523b47bdf) | head.S (84b04d3e6bdbc7551e62b75dd97cae4a8bddb1b6) |
---|---|
1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Low-level CPU initialisation 4 * Based on arch/arm/kernel/head.S 5 * 6 * Copyright (C) 1994-2002 Russell King 7 * Copyright (C) 2003-2012 ARM Ltd. 8 * Authors: Catalin Marinas <catalin.marinas@arm.com> --- 66 unchanged lines hidden (view full) --- 75 /* 76 * The following callee saved general purpose registers are used on the 77 * primary lowlevel boot path: 78 * 79 * Register Scope Purpose 80 * x19 primary_entry() .. start_kernel() whether we entered with the MMU on 81 * x20 primary_entry() .. __primary_switch() CPU boot mode 82 * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0 | 1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Low-level CPU initialisation 4 * Based on arch/arm/kernel/head.S 5 * 6 * Copyright (C) 1994-2002 Russell King 7 * Copyright (C) 2003-2012 ARM Ltd. 8 * Authors: Catalin Marinas <catalin.marinas@arm.com> --- 66 unchanged lines hidden (view full) --- 75 /* 76 * The following callee saved general purpose registers are used on the 77 * primary lowlevel boot path: 78 * 79 * Register Scope Purpose 80 * x19 primary_entry() .. start_kernel() whether we entered with the MMU on 81 * x20 primary_entry() .. __primary_switch() CPU boot mode 82 * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0 |
83 * x22 create_idmap() .. start_kernel() ID map VA of the DT blob | |
84 * x25 primary_entry() .. start_kernel() supported VA size | 83 * x25 primary_entry() .. start_kernel() supported VA size |
85 * x28 create_idmap() callee preserved temp register | |
86 */ 87SYM_CODE_START(primary_entry) 88 bl record_mmu_state 89 bl preserve_boot_args | 84 */ 85SYM_CODE_START(primary_entry) 86 bl record_mmu_state 87 bl preserve_boot_args |
90 bl create_idmap | |
91 | 88 |
89 adrp x1, early_init_stack 90 mov sp, x1 91 mov x29, xzr 92 adrp x0, init_idmap_pg_dir 93 bl __pi_create_init_idmap 94 |
|
92 /* | 95 /* |
96 * If the page tables have been populated with non-cacheable 97 * accesses (MMU disabled), invalidate those tables again to 98 * remove any speculatively loaded cache lines. 99 */ 100 cbnz x19, 0f 101 dmb sy 102 mov x1, x0 // end of used region 103 adrp x0, init_idmap_pg_dir 104 adr_l x2, dcache_inval_poc 105 blr x2 106 b 1f 107 108 /* |
|
93 * If we entered with the MMU and caches on, clean the ID mapped part 94 * of the primary boot code to the PoC so we can safely execute it with 95 * the MMU off. 96 */ | 109 * If we entered with the MMU and caches on, clean the ID mapped part 110 * of the primary boot code to the PoC so we can safely execute it with 111 * the MMU off. 112 */ |
97 cbz x19, 0f 98 adrp x0, __idmap_text_start | 1130: adrp x0, __idmap_text_start |
99 adr_l x1, __idmap_text_end 100 adr_l x2, dcache_clean_poc 101 blr x2 | 114 adr_l x1, __idmap_text_end 115 adr_l x2, dcache_clean_poc 116 blr x2 |
1020: mov x0, x19 | 117 1181: mov x0, x19 |
103 bl init_kernel_el // w0=cpu_boot_mode 104 mov x20, x0 105 106 /* 107 * The following calls CPU setup code, see arch/arm64/mm/proc.S for 108 * details. 109 * On return, the CPU will be ready for the MMU to be turned on and 110 * the TCR will have been set. --- 59 unchanged lines hidden (view full) --- 170 // MMU off 171 172 add x1, x0, #0x20 // 4 x 8 bytes 173 b dcache_inval_poc // tail call 1740: str_l x19, mmu_enabled_at_boot, x0 175 ret 176SYM_CODE_END(preserve_boot_args) 177 | 119 bl init_kernel_el // w0=cpu_boot_mode 120 mov x20, x0 121 122 /* 123 * The following calls CPU setup code, see arch/arm64/mm/proc.S for 124 * details. 125 * On return, the CPU will be ready for the MMU to be turned on and 126 * the TCR will have been set. --- 59 unchanged lines hidden (view full) --- 186 // MMU off 187 188 add x1, x0, #0x20 // 4 x 8 bytes 189 b dcache_inval_poc // tail call 1900: str_l x19, mmu_enabled_at_boot, x0 191 ret 192SYM_CODE_END(preserve_boot_args) 193 |
178/* 179 * Macro to populate page table entries, these entries can be pointers to the next level 180 * or last level entries pointing to physical memory. 181 * 182 * tbl: page table address 183 * rtbl: pointer to page table or physical memory 184 * index: start index to write 185 * eindex: end index to write - [index, eindex] written to 186 * flags: flags for pagetable entry to or in 187 * inc: increment to rtbl between each entry 188 * tmp1: temporary variable 189 * 190 * Preserves: tbl, eindex, flags, inc 191 * Corrupts: index, tmp1 192 * Returns: rtbl 193 */ 194 .macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1 195.Lpe\@: phys_to_pte \tmp1, \rtbl 196 orr \tmp1, \tmp1, \flags // tmp1 = table entry 197 str \tmp1, [\tbl, \index, lsl #3] 198 add \rtbl, \rtbl, \inc // rtbl = pa next level 199 add \index, \index, #1 200 cmp \index, \eindex 201 b.ls .Lpe\@ 202 .endm 203 204/* 205 * Compute indices of table entries from virtual address range. If multiple entries 206 * were needed in the previous page table level then the next page table level is assumed 207 * to be composed of multiple pages. (This effectively scales the end index). 208 * 209 * vstart: virtual address of start of range 210 * vend: virtual address of end of range - we map [vstart, vend] 211 * shift: shift used to transform virtual address into index 212 * order: #imm 2log(number of entries in page table) 213 * istart: index in table corresponding to vstart 214 * iend: index in table corresponding to vend 215 * count: On entry: how many extra entries were required in previous level, scales 216 * our end index. 217 * On exit: returns how many extra entries required for next page table level 218 * 219 * Preserves: vstart, vend 220 * Returns: istart, iend, count 221 */ 222 .macro compute_indices, vstart, vend, shift, order, istart, iend, count 223 ubfx \istart, \vstart, \shift, \order 224 ubfx \iend, \vend, \shift, \order 225 add \iend, \iend, \count, lsl \order 226 sub \count, \iend, \istart 227 .endm 228 229/* 230 * Map memory for specified virtual address range. Each level of page table needed supports 231 * multiple entries. If a level requires n entries the next page table level is assumed to be 232 * formed from n pages. 233 * 234 * tbl: location of page table 235 * rtbl: address to be used for first level page table entry (typically tbl + PAGE_SIZE) 236 * vstart: virtual address of start of range 237 * vend: virtual address of end of range - we map [vstart, vend - 1] 238 * flags: flags to use to map last level entries 239 * phys: physical address corresponding to vstart - physical memory is contiguous 240 * order: #imm 2log(number of entries in PGD table) 241 * 242 * If extra_shift is set, an extra level will be populated if the end address does 243 * not fit in 'extra_shift' bits. This assumes vend is in the TTBR0 range. 244 * 245 * Temporaries: istart, iend, tmp, count, sv - these need to be different registers 246 * Preserves: vstart, flags 247 * Corrupts: tbl, rtbl, vend, istart, iend, tmp, count, sv 248 */ 249 .macro map_memory, tbl, rtbl, vstart, vend, flags, phys, order, istart, iend, tmp, count, sv, extra_shift 250 sub \vend, \vend, #1 251 add \rtbl, \tbl, #PAGE_SIZE 252 mov \count, #0 253 254 .ifnb \extra_shift 255 tst \vend, #~((1 << (\extra_shift)) - 1) 256 b.eq .L_\@ 257 compute_indices \vstart, \vend, #\extra_shift, #(PAGE_SHIFT - 3), \istart, \iend, \count 258 mov \sv, \rtbl 259 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp 260 mov \tbl, \sv 261 .endif 262.L_\@: 263 compute_indices \vstart, \vend, #PGDIR_SHIFT, #\order, \istart, \iend, \count 264 mov \sv, \rtbl 265 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp 266 mov \tbl, \sv 267 268#if SWAPPER_PGTABLE_LEVELS > 3 269 compute_indices \vstart, \vend, #PUD_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count 270 mov \sv, \rtbl 271 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp 272 mov \tbl, \sv 273#endif 274 275#if SWAPPER_PGTABLE_LEVELS > 2 276 compute_indices \vstart, \vend, #SWAPPER_TABLE_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count 277 mov \sv, \rtbl 278 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp 279 mov \tbl, \sv 280#endif 281 282 compute_indices \vstart, \vend, #SWAPPER_BLOCK_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count 283 bic \rtbl, \phys, #SWAPPER_BLOCK_SIZE - 1 284 populate_entries \tbl, \rtbl, \istart, \iend, \flags, #SWAPPER_BLOCK_SIZE, \tmp 285 .endm 286 287/* 288 * Remap a subregion created with the map_memory macro with modified attributes 289 * or output address. The entire remapped region must have been covered in the 290 * invocation of map_memory. 291 * 292 * x0: last level table address (returned in first argument to map_memory) 293 * x1: start VA of the existing mapping 294 * x2: start VA of the region to update 295 * x3: end VA of the region to update (exclusive) 296 * x4: start PA associated with the region to update 297 * x5: attributes to set on the updated region 298 * x6: order of the last level mappings 299 */ 300SYM_FUNC_START_LOCAL(remap_region) 301 sub x3, x3, #1 // make end inclusive 302 303 // Get the index offset for the start of the last level table 304 lsr x1, x1, x6 305 bfi x1, xzr, #0, #PAGE_SHIFT - 3 306 307 // Derive the start and end indexes into the last level table 308 // associated with the provided region 309 lsr x2, x2, x6 310 lsr x3, x3, x6 311 sub x2, x2, x1 312 sub x3, x3, x1 313 314 mov x1, #1 315 lsl x6, x1, x6 // block size at this level 316 317 populate_entries x0, x4, x2, x3, x5, x6, x7 318 ret 319SYM_FUNC_END(remap_region) 320 321SYM_FUNC_START_LOCAL(create_idmap) 322 mov x28, lr | |
323 /* | 194 /* |
324 * The ID map carries a 1:1 mapping of the physical address range 325 * covered by the loaded image, which could be anywhere in DRAM. This 326 * means that the required size of the VA (== PA) space is decided at 327 * boot time, and could be more than the configured size of the VA 328 * space for ordinary kernel and user space mappings. 329 * 330 * There are three cases to consider here: 331 * - 39 <= VA_BITS < 48, and the ID map needs up to 48 VA bits to cover 332 * the placement of the image. In this case, we configure one extra 333 * level of translation on the fly for the ID map only. (This case 334 * also covers 42-bit VA/52-bit PA on 64k pages). 335 * 336 * - VA_BITS == 48, and the ID map needs more than 48 VA bits. This can 337 * only happen when using 64k pages, in which case we need to extend 338 * the root level table rather than add a level. Note that we can 339 * treat this case as 'always extended' as long as we take care not 340 * to program an unsupported T0SZ value into the TCR register. 341 * 342 * - Combinations that would require two additional levels of 343 * translation are not supported, e.g., VA_BITS==36 on 16k pages, or 344 * VA_BITS==39/4k pages with 5-level paging, where the input address 345 * requires more than 47 or 48 bits, respectively. 346 */ 347#if (VA_BITS < 48) 348#define IDMAP_PGD_ORDER (VA_BITS - PGDIR_SHIFT) 349#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) 350 351 /* 352 * If VA_BITS < 48, we have to configure an additional table level. 353 * First, we have to verify our assumption that the current value of 354 * VA_BITS was chosen such that all translation levels are fully 355 * utilised, and that lowering T0SZ will always result in an additional 356 * translation level to be configured. 357 */ 358#if VA_BITS != EXTRA_SHIFT 359#error "Mismatch between VA_BITS and page size/number of translation levels" 360#endif 361#else 362#define IDMAP_PGD_ORDER (PHYS_MASK_SHIFT - PGDIR_SHIFT) 363#define EXTRA_SHIFT 364 /* 365 * If VA_BITS == 48, we don't have to configure an additional 366 * translation level, but the top-level table has more entries. 367 */ 368#endif 369 adrp x0, init_idmap_pg_dir 370 adrp x3, _text 371 adrp x6, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE 372 mov_q x7, SWAPPER_RX_MMUFLAGS 373 374 map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT 375 376 /* Remap [.init].data, BSS and the kernel page tables r/w in the ID map */ 377 adrp x1, _text 378 adrp x2, __initdata_begin 379 adrp x3, _end 380 bic x4, x2, #SWAPPER_BLOCK_SIZE - 1 381 mov_q x5, SWAPPER_RW_MMUFLAGS 382 mov x6, #SWAPPER_BLOCK_SHIFT 383 bl remap_region 384 385 /* Remap the FDT after the kernel image */ 386 adrp x1, _text 387 adrp x22, _end + SWAPPER_BLOCK_SIZE 388 bic x2, x22, #SWAPPER_BLOCK_SIZE - 1 389 bfi x22, x21, #0, #SWAPPER_BLOCK_SHIFT // remapped FDT address 390 add x3, x2, #MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE 391 bic x4, x21, #SWAPPER_BLOCK_SIZE - 1 392 mov_q x5, SWAPPER_RW_MMUFLAGS 393 mov x6, #SWAPPER_BLOCK_SHIFT 394 bl remap_region 395 396 /* 397 * Since the page tables have been populated with non-cacheable 398 * accesses (MMU disabled), invalidate those tables again to 399 * remove any speculatively loaded cache lines. 400 */ 401 cbnz x19, 0f // skip cache invalidation if MMU is on 402 dmb sy 403 404 adrp x0, init_idmap_pg_dir 405 adrp x1, init_idmap_pg_end 406 bl dcache_inval_poc 4070: ret x28 408SYM_FUNC_END(create_idmap) 409 410 /* | |
411 * Initialize CPU registers with task-specific and cpu-specific context. 412 * 413 * Create a final frame record at task_pt_regs(current)->stackframe, so 414 * that the unwinder can identify the final frame record of any task by 415 * its location in the task stack. We reserve the entire pt_regs space 416 * for consistency with user tasks and kthreads. 417 */ 418 .macro init_cpu_task tsk, tmp1, tmp2 --- 305 unchanged lines hidden (view full) --- 724 CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2 7251: 726 wfe 727 wfi 728 b 1b 729SYM_FUNC_END(__no_granule_support) 730 731SYM_FUNC_START_LOCAL(__primary_switch) | 195 * Initialize CPU registers with task-specific and cpu-specific context. 196 * 197 * Create a final frame record at task_pt_regs(current)->stackframe, so 198 * that the unwinder can identify the final frame record of any task by 199 * its location in the task stack. We reserve the entire pt_regs space 200 * for consistency with user tasks and kthreads. 201 */ 202 .macro init_cpu_task tsk, tmp1, tmp2 --- 305 unchanged lines hidden (view full) --- 508 CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2 5091: 510 wfe 511 wfi 512 b 1b 513SYM_FUNC_END(__no_granule_support) 514 515SYM_FUNC_START_LOCAL(__primary_switch) |
732 mrs x1, tcr_el1 733 mov x2, #64 - VA_BITS 734 tcr_set_t0sz x1, x2 735 msr tcr_el1, x1 736 | |
737 adrp x1, reserved_pg_dir 738 adrp x2, init_idmap_pg_dir 739 bl __enable_mmu 740 741 adrp x1, early_init_stack 742 mov sp, x1 743 mov x29, xzr 744 mov x0, x20 // pass the full boot status | 516 adrp x1, reserved_pg_dir 517 adrp x2, init_idmap_pg_dir 518 bl __enable_mmu 519 520 adrp x1, early_init_stack 521 mov sp, x1 522 mov x29, xzr 523 mov x0, x20 // pass the full boot status |
745 mov x1, x22 // pass the low FDT mapping | 524 mov x1, x21 // pass the FDT |
746 bl __pi_early_map_kernel // Map and relocate the kernel 747 748 ldr x8, =__primary_switched 749 adrp x0, KERNEL_START // __pa(KERNEL_START) 750 br x8 751SYM_FUNC_END(__primary_switch) | 525 bl __pi_early_map_kernel // Map and relocate the kernel 526 527 ldr x8, =__primary_switched 528 adrp x0, KERNEL_START // __pa(KERNEL_START) 529 br x8 530SYM_FUNC_END(__primary_switch) |