head.S (e6128a8e523cfa8f46a1501e6432c37523b47bdf) head.S (84b04d3e6bdbc7551e62b75dd97cae4a8bddb1b6)
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Low-level CPU initialisation
4 * Based on arch/arm/kernel/head.S
5 *
6 * Copyright (C) 1994-2002 Russell King
7 * Copyright (C) 2003-2012 ARM Ltd.
8 * Authors: Catalin Marinas <catalin.marinas@arm.com>

--- 66 unchanged lines hidden (view full) ---

75 /*
76 * The following callee saved general purpose registers are used on the
77 * primary lowlevel boot path:
78 *
79 * Register Scope Purpose
80 * x19 primary_entry() .. start_kernel() whether we entered with the MMU on
81 * x20 primary_entry() .. __primary_switch() CPU boot mode
82 * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Low-level CPU initialisation
4 * Based on arch/arm/kernel/head.S
5 *
6 * Copyright (C) 1994-2002 Russell King
7 * Copyright (C) 2003-2012 ARM Ltd.
8 * Authors: Catalin Marinas <catalin.marinas@arm.com>

--- 66 unchanged lines hidden (view full) ---

75 /*
76 * The following callee saved general purpose registers are used on the
77 * primary lowlevel boot path:
78 *
79 * Register Scope Purpose
80 * x19 primary_entry() .. start_kernel() whether we entered with the MMU on
81 * x20 primary_entry() .. __primary_switch() CPU boot mode
82 * x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0
83 * x22 create_idmap() .. start_kernel() ID map VA of the DT blob
84 * x25 primary_entry() .. start_kernel() supported VA size
83 * x25 primary_entry() .. start_kernel() supported VA size
85 * x28 create_idmap() callee preserved temp register
86 */
87SYM_CODE_START(primary_entry)
88 bl record_mmu_state
89 bl preserve_boot_args
84 */
85SYM_CODE_START(primary_entry)
86 bl record_mmu_state
87 bl preserve_boot_args
90 bl create_idmap
91
88
89 adrp x1, early_init_stack
90 mov sp, x1
91 mov x29, xzr
92 adrp x0, init_idmap_pg_dir
93 bl __pi_create_init_idmap
94
92 /*
95 /*
96 * If the page tables have been populated with non-cacheable
97 * accesses (MMU disabled), invalidate those tables again to
98 * remove any speculatively loaded cache lines.
99 */
100 cbnz x19, 0f
101 dmb sy
102 mov x1, x0 // end of used region
103 adrp x0, init_idmap_pg_dir
104 adr_l x2, dcache_inval_poc
105 blr x2
106 b 1f
107
108 /*
93 * If we entered with the MMU and caches on, clean the ID mapped part
94 * of the primary boot code to the PoC so we can safely execute it with
95 * the MMU off.
96 */
109 * If we entered with the MMU and caches on, clean the ID mapped part
110 * of the primary boot code to the PoC so we can safely execute it with
111 * the MMU off.
112 */
97 cbz x19, 0f
98 adrp x0, __idmap_text_start
1130: adrp x0, __idmap_text_start
99 adr_l x1, __idmap_text_end
100 adr_l x2, dcache_clean_poc
101 blr x2
114 adr_l x1, __idmap_text_end
115 adr_l x2, dcache_clean_poc
116 blr x2
1020: mov x0, x19
117
1181: mov x0, x19
103 bl init_kernel_el // w0=cpu_boot_mode
104 mov x20, x0
105
106 /*
107 * The following calls CPU setup code, see arch/arm64/mm/proc.S for
108 * details.
109 * On return, the CPU will be ready for the MMU to be turned on and
110 * the TCR will have been set.

--- 59 unchanged lines hidden (view full) ---

170 // MMU off
171
172 add x1, x0, #0x20 // 4 x 8 bytes
173 b dcache_inval_poc // tail call
1740: str_l x19, mmu_enabled_at_boot, x0
175 ret
176SYM_CODE_END(preserve_boot_args)
177
119 bl init_kernel_el // w0=cpu_boot_mode
120 mov x20, x0
121
122 /*
123 * The following calls CPU setup code, see arch/arm64/mm/proc.S for
124 * details.
125 * On return, the CPU will be ready for the MMU to be turned on and
126 * the TCR will have been set.

--- 59 unchanged lines hidden (view full) ---

186 // MMU off
187
188 add x1, x0, #0x20 // 4 x 8 bytes
189 b dcache_inval_poc // tail call
1900: str_l x19, mmu_enabled_at_boot, x0
191 ret
192SYM_CODE_END(preserve_boot_args)
193
178/*
179 * Macro to populate page table entries, these entries can be pointers to the next level
180 * or last level entries pointing to physical memory.
181 *
182 * tbl: page table address
183 * rtbl: pointer to page table or physical memory
184 * index: start index to write
185 * eindex: end index to write - [index, eindex] written to
186 * flags: flags for pagetable entry to or in
187 * inc: increment to rtbl between each entry
188 * tmp1: temporary variable
189 *
190 * Preserves: tbl, eindex, flags, inc
191 * Corrupts: index, tmp1
192 * Returns: rtbl
193 */
194 .macro populate_entries, tbl, rtbl, index, eindex, flags, inc, tmp1
195.Lpe\@: phys_to_pte \tmp1, \rtbl
196 orr \tmp1, \tmp1, \flags // tmp1 = table entry
197 str \tmp1, [\tbl, \index, lsl #3]
198 add \rtbl, \rtbl, \inc // rtbl = pa next level
199 add \index, \index, #1
200 cmp \index, \eindex
201 b.ls .Lpe\@
202 .endm
203
204/*
205 * Compute indices of table entries from virtual address range. If multiple entries
206 * were needed in the previous page table level then the next page table level is assumed
207 * to be composed of multiple pages. (This effectively scales the end index).
208 *
209 * vstart: virtual address of start of range
210 * vend: virtual address of end of range - we map [vstart, vend]
211 * shift: shift used to transform virtual address into index
212 * order: #imm 2log(number of entries in page table)
213 * istart: index in table corresponding to vstart
214 * iend: index in table corresponding to vend
215 * count: On entry: how many extra entries were required in previous level, scales
216 * our end index.
217 * On exit: returns how many extra entries required for next page table level
218 *
219 * Preserves: vstart, vend
220 * Returns: istart, iend, count
221 */
222 .macro compute_indices, vstart, vend, shift, order, istart, iend, count
223 ubfx \istart, \vstart, \shift, \order
224 ubfx \iend, \vend, \shift, \order
225 add \iend, \iend, \count, lsl \order
226 sub \count, \iend, \istart
227 .endm
228
229/*
230 * Map memory for specified virtual address range. Each level of page table needed supports
231 * multiple entries. If a level requires n entries the next page table level is assumed to be
232 * formed from n pages.
233 *
234 * tbl: location of page table
235 * rtbl: address to be used for first level page table entry (typically tbl + PAGE_SIZE)
236 * vstart: virtual address of start of range
237 * vend: virtual address of end of range - we map [vstart, vend - 1]
238 * flags: flags to use to map last level entries
239 * phys: physical address corresponding to vstart - physical memory is contiguous
240 * order: #imm 2log(number of entries in PGD table)
241 *
242 * If extra_shift is set, an extra level will be populated if the end address does
243 * not fit in 'extra_shift' bits. This assumes vend is in the TTBR0 range.
244 *
245 * Temporaries: istart, iend, tmp, count, sv - these need to be different registers
246 * Preserves: vstart, flags
247 * Corrupts: tbl, rtbl, vend, istart, iend, tmp, count, sv
248 */
249 .macro map_memory, tbl, rtbl, vstart, vend, flags, phys, order, istart, iend, tmp, count, sv, extra_shift
250 sub \vend, \vend, #1
251 add \rtbl, \tbl, #PAGE_SIZE
252 mov \count, #0
253
254 .ifnb \extra_shift
255 tst \vend, #~((1 << (\extra_shift)) - 1)
256 b.eq .L_\@
257 compute_indices \vstart, \vend, #\extra_shift, #(PAGE_SHIFT - 3), \istart, \iend, \count
258 mov \sv, \rtbl
259 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
260 mov \tbl, \sv
261 .endif
262.L_\@:
263 compute_indices \vstart, \vend, #PGDIR_SHIFT, #\order, \istart, \iend, \count
264 mov \sv, \rtbl
265 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
266 mov \tbl, \sv
267
268#if SWAPPER_PGTABLE_LEVELS > 3
269 compute_indices \vstart, \vend, #PUD_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
270 mov \sv, \rtbl
271 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
272 mov \tbl, \sv
273#endif
274
275#if SWAPPER_PGTABLE_LEVELS > 2
276 compute_indices \vstart, \vend, #SWAPPER_TABLE_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
277 mov \sv, \rtbl
278 populate_entries \tbl, \rtbl, \istart, \iend, #PMD_TYPE_TABLE, #PAGE_SIZE, \tmp
279 mov \tbl, \sv
280#endif
281
282 compute_indices \vstart, \vend, #SWAPPER_BLOCK_SHIFT, #(PAGE_SHIFT - 3), \istart, \iend, \count
283 bic \rtbl, \phys, #SWAPPER_BLOCK_SIZE - 1
284 populate_entries \tbl, \rtbl, \istart, \iend, \flags, #SWAPPER_BLOCK_SIZE, \tmp
285 .endm
286
287/*
288 * Remap a subregion created with the map_memory macro with modified attributes
289 * or output address. The entire remapped region must have been covered in the
290 * invocation of map_memory.
291 *
292 * x0: last level table address (returned in first argument to map_memory)
293 * x1: start VA of the existing mapping
294 * x2: start VA of the region to update
295 * x3: end VA of the region to update (exclusive)
296 * x4: start PA associated with the region to update
297 * x5: attributes to set on the updated region
298 * x6: order of the last level mappings
299 */
300SYM_FUNC_START_LOCAL(remap_region)
301 sub x3, x3, #1 // make end inclusive
302
303 // Get the index offset for the start of the last level table
304 lsr x1, x1, x6
305 bfi x1, xzr, #0, #PAGE_SHIFT - 3
306
307 // Derive the start and end indexes into the last level table
308 // associated with the provided region
309 lsr x2, x2, x6
310 lsr x3, x3, x6
311 sub x2, x2, x1
312 sub x3, x3, x1
313
314 mov x1, #1
315 lsl x6, x1, x6 // block size at this level
316
317 populate_entries x0, x4, x2, x3, x5, x6, x7
318 ret
319SYM_FUNC_END(remap_region)
320
321SYM_FUNC_START_LOCAL(create_idmap)
322 mov x28, lr
323 /*
194 /*
324 * The ID map carries a 1:1 mapping of the physical address range
325 * covered by the loaded image, which could be anywhere in DRAM. This
326 * means that the required size of the VA (== PA) space is decided at
327 * boot time, and could be more than the configured size of the VA
328 * space for ordinary kernel and user space mappings.
329 *
330 * There are three cases to consider here:
331 * - 39 <= VA_BITS < 48, and the ID map needs up to 48 VA bits to cover
332 * the placement of the image. In this case, we configure one extra
333 * level of translation on the fly for the ID map only. (This case
334 * also covers 42-bit VA/52-bit PA on 64k pages).
335 *
336 * - VA_BITS == 48, and the ID map needs more than 48 VA bits. This can
337 * only happen when using 64k pages, in which case we need to extend
338 * the root level table rather than add a level. Note that we can
339 * treat this case as 'always extended' as long as we take care not
340 * to program an unsupported T0SZ value into the TCR register.
341 *
342 * - Combinations that would require two additional levels of
343 * translation are not supported, e.g., VA_BITS==36 on 16k pages, or
344 * VA_BITS==39/4k pages with 5-level paging, where the input address
345 * requires more than 47 or 48 bits, respectively.
346 */
347#if (VA_BITS < 48)
348#define IDMAP_PGD_ORDER (VA_BITS - PGDIR_SHIFT)
349#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3)
350
351 /*
352 * If VA_BITS < 48, we have to configure an additional table level.
353 * First, we have to verify our assumption that the current value of
354 * VA_BITS was chosen such that all translation levels are fully
355 * utilised, and that lowering T0SZ will always result in an additional
356 * translation level to be configured.
357 */
358#if VA_BITS != EXTRA_SHIFT
359#error "Mismatch between VA_BITS and page size/number of translation levels"
360#endif
361#else
362#define IDMAP_PGD_ORDER (PHYS_MASK_SHIFT - PGDIR_SHIFT)
363#define EXTRA_SHIFT
364 /*
365 * If VA_BITS == 48, we don't have to configure an additional
366 * translation level, but the top-level table has more entries.
367 */
368#endif
369 adrp x0, init_idmap_pg_dir
370 adrp x3, _text
371 adrp x6, _end + MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE
372 mov_q x7, SWAPPER_RX_MMUFLAGS
373
374 map_memory x0, x1, x3, x6, x7, x3, IDMAP_PGD_ORDER, x10, x11, x12, x13, x14, EXTRA_SHIFT
375
376 /* Remap [.init].data, BSS and the kernel page tables r/w in the ID map */
377 adrp x1, _text
378 adrp x2, __initdata_begin
379 adrp x3, _end
380 bic x4, x2, #SWAPPER_BLOCK_SIZE - 1
381 mov_q x5, SWAPPER_RW_MMUFLAGS
382 mov x6, #SWAPPER_BLOCK_SHIFT
383 bl remap_region
384
385 /* Remap the FDT after the kernel image */
386 adrp x1, _text
387 adrp x22, _end + SWAPPER_BLOCK_SIZE
388 bic x2, x22, #SWAPPER_BLOCK_SIZE - 1
389 bfi x22, x21, #0, #SWAPPER_BLOCK_SHIFT // remapped FDT address
390 add x3, x2, #MAX_FDT_SIZE + SWAPPER_BLOCK_SIZE
391 bic x4, x21, #SWAPPER_BLOCK_SIZE - 1
392 mov_q x5, SWAPPER_RW_MMUFLAGS
393 mov x6, #SWAPPER_BLOCK_SHIFT
394 bl remap_region
395
396 /*
397 * Since the page tables have been populated with non-cacheable
398 * accesses (MMU disabled), invalidate those tables again to
399 * remove any speculatively loaded cache lines.
400 */
401 cbnz x19, 0f // skip cache invalidation if MMU is on
402 dmb sy
403
404 adrp x0, init_idmap_pg_dir
405 adrp x1, init_idmap_pg_end
406 bl dcache_inval_poc
4070: ret x28
408SYM_FUNC_END(create_idmap)
409
410 /*
411 * Initialize CPU registers with task-specific and cpu-specific context.
412 *
413 * Create a final frame record at task_pt_regs(current)->stackframe, so
414 * that the unwinder can identify the final frame record of any task by
415 * its location in the task stack. We reserve the entire pt_regs space
416 * for consistency with user tasks and kthreads.
417 */
418 .macro init_cpu_task tsk, tmp1, tmp2

--- 305 unchanged lines hidden (view full) ---

724 CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2
7251:
726 wfe
727 wfi
728 b 1b
729SYM_FUNC_END(__no_granule_support)
730
731SYM_FUNC_START_LOCAL(__primary_switch)
195 * Initialize CPU registers with task-specific and cpu-specific context.
196 *
197 * Create a final frame record at task_pt_regs(current)->stackframe, so
198 * that the unwinder can identify the final frame record of any task by
199 * its location in the task stack. We reserve the entire pt_regs space
200 * for consistency with user tasks and kthreads.
201 */
202 .macro init_cpu_task tsk, tmp1, tmp2

--- 305 unchanged lines hidden (view full) ---

508 CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2
5091:
510 wfe
511 wfi
512 b 1b
513SYM_FUNC_END(__no_granule_support)
514
515SYM_FUNC_START_LOCAL(__primary_switch)
732 mrs x1, tcr_el1
733 mov x2, #64 - VA_BITS
734 tcr_set_t0sz x1, x2
735 msr tcr_el1, x1
736
737 adrp x1, reserved_pg_dir
738 adrp x2, init_idmap_pg_dir
739 bl __enable_mmu
740
741 adrp x1, early_init_stack
742 mov sp, x1
743 mov x29, xzr
744 mov x0, x20 // pass the full boot status
516 adrp x1, reserved_pg_dir
517 adrp x2, init_idmap_pg_dir
518 bl __enable_mmu
519
520 adrp x1, early_init_stack
521 mov sp, x1
522 mov x29, xzr
523 mov x0, x20 // pass the full boot status
745 mov x1, x22 // pass the low FDT mapping
524 mov x1, x21 // pass the FDT
746 bl __pi_early_map_kernel // Map and relocate the kernel
747
748 ldr x8, =__primary_switched
749 adrp x0, KERNEL_START // __pa(KERNEL_START)
750 br x8
751SYM_FUNC_END(__primary_switch)
525 bl __pi_early_map_kernel // Map and relocate the kernel
526
527 ldr x8, =__primary_switched
528 adrp x0, KERNEL_START // __pa(KERNEL_START)
529 br x8
530SYM_FUNC_END(__primary_switch)