xref: /linux/arch/arm64/mm/proc.S (revision d6e4b3e326d8b44675b9e19534347d97073826aa) 
1/*
2 * Based on arch/arm/mm/proc.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#include <linux/init.h>
22#include <linux/linkage.h>
23#include <asm/assembler.h>
24#include <asm/asm-offsets.h>
25#include <asm/hwcap.h>
26#include <asm/pgtable.h>
27#include <asm/pgtable-hwdef.h>
28#include <asm/cpufeature.h>
29#include <asm/alternative.h>
30
31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
33#elif defined(CONFIG_ARM64_16K_PAGES)
34#define TCR_TG_FLAGS	TCR_TG0_16K | TCR_TG1_16K
35#else /* CONFIG_ARM64_4K_PAGES */
36#define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
37#endif
38
39#ifdef CONFIG_RANDOMIZE_BASE
40#define TCR_KASLR_FLAGS	TCR_NFD1
41#else
42#define TCR_KASLR_FLAGS	0
43#endif
44
45#define TCR_SMP_FLAGS	TCR_SHARED
46
47/* PTWs cacheable, inner/outer WBWA */
48#define TCR_CACHE_FLAGS	TCR_IRGN_WBWA | TCR_ORGN_WBWA
49
50#ifdef CONFIG_KASAN_SW_TAGS
51#define TCR_KASAN_FLAGS TCR_TBI1
52#else
53#define TCR_KASAN_FLAGS 0
54#endif
55
56#define MAIR(attr, mt)	((attr) << ((mt) * 8))
57
58/*
59 *	cpu_do_idle()
60 *
61 *	Idle the processor (wait for interrupt).
62 */
63ENTRY(cpu_do_idle)
64	dsb	sy				// WFI may enter a low-power mode
65	wfi
66	ret
67ENDPROC(cpu_do_idle)
68
69#ifdef CONFIG_CPU_PM
70/**
71 * cpu_do_suspend - save CPU registers context
72 *
73 * x0: virtual address of context pointer
74 */
75ENTRY(cpu_do_suspend)
76	mrs	x2, tpidr_el0
77	mrs	x3, tpidrro_el0
78	mrs	x4, contextidr_el1
79	mrs	x5, cpacr_el1
80	mrs	x6, tcr_el1
81	mrs	x7, vbar_el1
82	mrs	x8, mdscr_el1
83	mrs	x9, oslsr_el1
84	mrs	x10, sctlr_el1
85alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
86	mrs	x11, tpidr_el1
87alternative_else
88	mrs	x11, tpidr_el2
89alternative_endif
90	mrs	x12, sp_el0
91	stp	x2, x3, [x0]
92	stp	x4, xzr, [x0, #16]
93	stp	x5, x6, [x0, #32]
94	stp	x7, x8, [x0, #48]
95	stp	x9, x10, [x0, #64]
96	stp	x11, x12, [x0, #80]
97	ret
98ENDPROC(cpu_do_suspend)
99
100/**
101 * cpu_do_resume - restore CPU register context
102 *
103 * x0: Address of context pointer
104 */
105	.pushsection ".idmap.text", "awx"
106ENTRY(cpu_do_resume)
107	ldp	x2, x3, [x0]
108	ldp	x4, x5, [x0, #16]
109	ldp	x6, x8, [x0, #32]
110	ldp	x9, x10, [x0, #48]
111	ldp	x11, x12, [x0, #64]
112	ldp	x13, x14, [x0, #80]
113	msr	tpidr_el0, x2
114	msr	tpidrro_el0, x3
115	msr	contextidr_el1, x4
116	msr	cpacr_el1, x6
117
118	/* Don't change t0sz here, mask those bits when restoring */
119	mrs	x5, tcr_el1
120	bfi	x8, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
121
122	msr	tcr_el1, x8
123	msr	vbar_el1, x9
124
125	/*
126	 * __cpu_setup() cleared MDSCR_EL1.MDE and friends, before unmasking
127	 * debug exceptions. By restoring MDSCR_EL1 here, we may take a debug
128	 * exception. Mask them until local_daif_restore() in cpu_suspend()
129	 * resets them.
130	 */
131	disable_daif
132	msr	mdscr_el1, x10
133
134	msr	sctlr_el1, x12
135alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
136	msr	tpidr_el1, x13
137alternative_else
138	msr	tpidr_el2, x13
139alternative_endif
140	msr	sp_el0, x14
141	/*
142	 * Restore oslsr_el1 by writing oslar_el1
143	 */
144	ubfx	x11, x11, #1, #1
145	msr	oslar_el1, x11
146	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
147
148alternative_if ARM64_HAS_RAS_EXTN
149	msr_s	SYS_DISR_EL1, xzr
150alternative_else_nop_endif
151
152	isb
153	ret
154ENDPROC(cpu_do_resume)
155	.popsection
156#endif
157
158/*
159 *	cpu_do_switch_mm(pgd_phys, tsk)
160 *
161 *	Set the translation table base pointer to be pgd_phys.
162 *
163 *	- pgd_phys - physical address of new TTB
164 */
165ENTRY(cpu_do_switch_mm)
166	mrs	x2, ttbr1_el1
167	mmid	x1, x1				// get mm->context.id
168	phys_to_ttbr x3, x0
169
170alternative_if ARM64_HAS_CNP
171	cbz     x1, 1f                          // skip CNP for reserved ASID
172	orr     x3, x3, #TTBR_CNP_BIT
1731:
174alternative_else_nop_endif
175#ifdef CONFIG_ARM64_SW_TTBR0_PAN
176	bfi	x3, x1, #48, #16		// set the ASID field in TTBR0
177#endif
178	bfi	x2, x1, #48, #16		// set the ASID
179	msr	ttbr1_el1, x2			// in TTBR1 (since TCR.A1 is set)
180	isb
181	msr	ttbr0_el1, x3			// now update TTBR0
182	isb
183	b	post_ttbr_update_workaround	// Back to C code...
184ENDPROC(cpu_do_switch_mm)
185
186	.pushsection ".idmap.text", "awx"
187
188.macro	__idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
189	adrp	\tmp1, empty_zero_page
190	phys_to_ttbr \tmp2, \tmp1
191	offset_ttbr1 \tmp2
192	msr	ttbr1_el1, \tmp2
193	isb
194	tlbi	vmalle1
195	dsb	nsh
196	isb
197.endm
198
199/*
200 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1)
201 *
202 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
203 * called by anything else. It can only be executed from a TTBR0 mapping.
204 */
205ENTRY(idmap_cpu_replace_ttbr1)
206	save_and_disable_daif flags=x2
207
208	__idmap_cpu_set_reserved_ttbr1 x1, x3
209
210	offset_ttbr1 x0
211	msr	ttbr1_el1, x0
212	isb
213
214	restore_daif x2
215
216	ret
217ENDPROC(idmap_cpu_replace_ttbr1)
218	.popsection
219
220#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
221	.pushsection ".idmap.text", "awx"
222
223	.macro	__idmap_kpti_get_pgtable_ent, type
224	dc	cvac, cur_\()\type\()p		// Ensure any existing dirty
225	dmb	sy				// lines are written back before
226	ldr	\type, [cur_\()\type\()p]	// loading the entry
227	tbz	\type, #0, skip_\()\type	// Skip invalid and
228	tbnz	\type, #11, skip_\()\type	// non-global entries
229	.endm
230
231	.macro __idmap_kpti_put_pgtable_ent_ng, type
232	orr	\type, \type, #PTE_NG		// Same bit for blocks and pages
233	str	\type, [cur_\()\type\()p]	// Update the entry and ensure
234	dmb	sy				// that it is visible to all
235	dc	civac, cur_\()\type\()p		// CPUs.
236	.endm
237
238/*
239 * void __kpti_install_ng_mappings(int cpu, int num_cpus, phys_addr_t swapper)
240 *
241 * Called exactly once from stop_machine context by each CPU found during boot.
242 */
243__idmap_kpti_flag:
244	.long	1
245ENTRY(idmap_kpti_install_ng_mappings)
246	cpu		.req	w0
247	num_cpus	.req	w1
248	swapper_pa	.req	x2
249	swapper_ttb	.req	x3
250	flag_ptr	.req	x4
251	cur_pgdp	.req	x5
252	end_pgdp	.req	x6
253	pgd		.req	x7
254	cur_pudp	.req	x8
255	end_pudp	.req	x9
256	pud		.req	x10
257	cur_pmdp	.req	x11
258	end_pmdp	.req	x12
259	pmd		.req	x13
260	cur_ptep	.req	x14
261	end_ptep	.req	x15
262	pte		.req	x16
263
264	mrs	swapper_ttb, ttbr1_el1
265	restore_ttbr1	swapper_ttb
266	adr	flag_ptr, __idmap_kpti_flag
267
268	cbnz	cpu, __idmap_kpti_secondary
269
270	/* We're the boot CPU. Wait for the others to catch up */
271	sevl
2721:	wfe
273	ldaxr	w18, [flag_ptr]
274	eor	w18, w18, num_cpus
275	cbnz	w18, 1b
276
277	/* We need to walk swapper, so turn off the MMU. */
278	pre_disable_mmu_workaround
279	mrs	x18, sctlr_el1
280	bic	x18, x18, #SCTLR_ELx_M
281	msr	sctlr_el1, x18
282	isb
283
284	/* Everybody is enjoying the idmap, so we can rewrite swapper. */
285	/* PGD */
286	mov	cur_pgdp, swapper_pa
287	add	end_pgdp, cur_pgdp, #(PTRS_PER_PGD * 8)
288do_pgd:	__idmap_kpti_get_pgtable_ent	pgd
289	tbnz	pgd, #1, walk_puds
290next_pgd:
291	__idmap_kpti_put_pgtable_ent_ng	pgd
292skip_pgd:
293	add	cur_pgdp, cur_pgdp, #8
294	cmp	cur_pgdp, end_pgdp
295	b.ne	do_pgd
296
297	/* Publish the updated tables and nuke all the TLBs */
298	dsb	sy
299	tlbi	vmalle1is
300	dsb	ish
301	isb
302
303	/* We're done: fire up the MMU again */
304	mrs	x18, sctlr_el1
305	orr	x18, x18, #SCTLR_ELx_M
306	msr	sctlr_el1, x18
307	isb
308
309	/* Set the flag to zero to indicate that we're all done */
310	str	wzr, [flag_ptr]
311	ret
312
313	/* PUD */
314walk_puds:
315	.if CONFIG_PGTABLE_LEVELS > 3
316	pte_to_phys	cur_pudp, pgd
317	add	end_pudp, cur_pudp, #(PTRS_PER_PUD * 8)
318do_pud:	__idmap_kpti_get_pgtable_ent	pud
319	tbnz	pud, #1, walk_pmds
320next_pud:
321	__idmap_kpti_put_pgtable_ent_ng	pud
322skip_pud:
323	add	cur_pudp, cur_pudp, 8
324	cmp	cur_pudp, end_pudp
325	b.ne	do_pud
326	b	next_pgd
327	.else /* CONFIG_PGTABLE_LEVELS <= 3 */
328	mov	pud, pgd
329	b	walk_pmds
330next_pud:
331	b	next_pgd
332	.endif
333
334	/* PMD */
335walk_pmds:
336	.if CONFIG_PGTABLE_LEVELS > 2
337	pte_to_phys	cur_pmdp, pud
338	add	end_pmdp, cur_pmdp, #(PTRS_PER_PMD * 8)
339do_pmd:	__idmap_kpti_get_pgtable_ent	pmd
340	tbnz	pmd, #1, walk_ptes
341next_pmd:
342	__idmap_kpti_put_pgtable_ent_ng	pmd
343skip_pmd:
344	add	cur_pmdp, cur_pmdp, #8
345	cmp	cur_pmdp, end_pmdp
346	b.ne	do_pmd
347	b	next_pud
348	.else /* CONFIG_PGTABLE_LEVELS <= 2 */
349	mov	pmd, pud
350	b	walk_ptes
351next_pmd:
352	b	next_pud
353	.endif
354
355	/* PTE */
356walk_ptes:
357	pte_to_phys	cur_ptep, pmd
358	add	end_ptep, cur_ptep, #(PTRS_PER_PTE * 8)
359do_pte:	__idmap_kpti_get_pgtable_ent	pte
360	__idmap_kpti_put_pgtable_ent_ng	pte
361skip_pte:
362	add	cur_ptep, cur_ptep, #8
363	cmp	cur_ptep, end_ptep
364	b.ne	do_pte
365	b	next_pmd
366
367	/* Secondary CPUs end up here */
368__idmap_kpti_secondary:
369	/* Uninstall swapper before surgery begins */
370	__idmap_cpu_set_reserved_ttbr1 x18, x17
371
372	/* Increment the flag to let the boot CPU we're ready */
3731:	ldxr	w18, [flag_ptr]
374	add	w18, w18, #1
375	stxr	w17, w18, [flag_ptr]
376	cbnz	w17, 1b
377
378	/* Wait for the boot CPU to finish messing around with swapper */
379	sevl
3801:	wfe
381	ldxr	w18, [flag_ptr]
382	cbnz	w18, 1b
383
384	/* All done, act like nothing happened */
385	offset_ttbr1 swapper_ttb
386	msr	ttbr1_el1, swapper_ttb
387	isb
388	ret
389
390	.unreq	cpu
391	.unreq	num_cpus
392	.unreq	swapper_pa
393	.unreq	swapper_ttb
394	.unreq	flag_ptr
395	.unreq	cur_pgdp
396	.unreq	end_pgdp
397	.unreq	pgd
398	.unreq	cur_pudp
399	.unreq	end_pudp
400	.unreq	pud
401	.unreq	cur_pmdp
402	.unreq	end_pmdp
403	.unreq	pmd
404	.unreq	cur_ptep
405	.unreq	end_ptep
406	.unreq	pte
407ENDPROC(idmap_kpti_install_ng_mappings)
408	.popsection
409#endif
410
411/*
412 *	__cpu_setup
413 *
414 *	Initialise the processor for turning the MMU on.  Return in x0 the
415 *	value of the SCTLR_EL1 register.
416 */
417	.pushsection ".idmap.text", "awx"
418ENTRY(__cpu_setup)
419	tlbi	vmalle1				// Invalidate local TLB
420	dsb	nsh
421
422	mov	x0, #3 << 20
423	msr	cpacr_el1, x0			// Enable FP/ASIMD
424	mov	x0, #1 << 12			// Reset mdscr_el1 and disable
425	msr	mdscr_el1, x0			// access to the DCC from EL0
426	isb					// Unmask debug exceptions now,
427	enable_dbg				// since this is per-cpu
428	reset_pmuserenr_el0 x0			// Disable PMU access from EL0
429	/*
430	 * Memory region attributes for LPAE:
431	 *
432	 *   n = AttrIndx[2:0]
433	 *			n	MAIR
434	 *   DEVICE_nGnRnE	000	00000000
435	 *   DEVICE_nGnRE	001	00000100
436	 *   DEVICE_GRE		010	00001100
437	 *   NORMAL_NC		011	01000100
438	 *   NORMAL		100	11111111
439	 *   NORMAL_WT		101	10111011
440	 */
441	ldr	x5, =MAIR(0x00, MT_DEVICE_nGnRnE) | \
442		     MAIR(0x04, MT_DEVICE_nGnRE) | \
443		     MAIR(0x0c, MT_DEVICE_GRE) | \
444		     MAIR(0x44, MT_NORMAL_NC) | \
445		     MAIR(0xff, MT_NORMAL) | \
446		     MAIR(0xbb, MT_NORMAL_WT)
447	msr	mair_el1, x5
448	/*
449	 * Prepare SCTLR
450	 */
451	mov_q	x0, SCTLR_EL1_SET
452	/*
453	 * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
454	 * both user and kernel.
455	 */
456	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
457			TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
458			TCR_TBI0 | TCR_A1 | TCR_KASAN_FLAGS
459
460#ifdef CONFIG_ARM64_USER_VA_BITS_52
461	ldr_l		x9, vabits_user
462	sub		x9, xzr, x9
463	add		x9, x9, #64
464#else
465	ldr_l		x9, idmap_t0sz
466#endif
467	tcr_set_t0sz	x10, x9
468
469	/*
470	 * Set the IPS bits in TCR_EL1.
471	 */
472	tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6
473#ifdef CONFIG_ARM64_HW_AFDBM
474	/*
475	 * Enable hardware update of the Access Flags bit.
476	 * Hardware dirty bit management is enabled later,
477	 * via capabilities.
478	 */
479	mrs	x9, ID_AA64MMFR1_EL1
480	and	x9, x9, #0xf
481	cbz	x9, 1f
482	orr	x10, x10, #TCR_HA		// hardware Access flag update
4831:
484#endif	/* CONFIG_ARM64_HW_AFDBM */
485	msr	tcr_el1, x10
486	ret					// return to head.S
487ENDPROC(__cpu_setup)
488