xref: /linux/arch/sparc/kernel/ktlb.S (revision 24bce201d79807b668bf9d9e0aca801c5c0d5f78) 
1/* SPDX-License-Identifier: GPL-2.0 */
2/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
3 *
4 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
5 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
6 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
7 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
8 */
9
10#include <linux/pgtable.h>
11#include <asm/head.h>
12#include <asm/asi.h>
13#include <asm/page.h>
14#include <asm/tsb.h>
15
16	.text
17	.align		32
18
19kvmap_itlb:
20	/* g6: TAG TARGET */
21	mov		TLB_TAG_ACCESS, %g4
22	ldxa		[%g4] ASI_IMMU, %g4
23
24	/* The kernel executes in context zero, therefore we do not
25	 * need to clear the context ID bits out of %g4 here.
26	 */
27
28	/* sun4v_itlb_miss branches here with the missing virtual
29	 * address already loaded into %g4
30	 */
31kvmap_itlb_4v:
32
33	/* Catch kernel NULL pointer calls.  */
34	sethi		%hi(PAGE_SIZE), %g5
35	cmp		%g4, %g5
36	blu,pn		%xcc, kvmap_itlb_longpath
37	 nop
38
39	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
40
41kvmap_itlb_tsb_miss:
42	sethi		%hi(LOW_OBP_ADDRESS), %g5
43	cmp		%g4, %g5
44	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
45	 mov		0x1, %g5
46	sllx		%g5, 32, %g5
47	cmp		%g4, %g5
48	blu,pn		%xcc, kvmap_itlb_obp
49	 nop
50
51kvmap_itlb_vmalloc_addr:
52	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
53
54	TSB_LOCK_TAG(%g1, %g2, %g7)
55	TSB_WRITE(%g1, %g5, %g6)
56
57	/* fallthrough to TLB load */
58
59kvmap_itlb_load:
60
61661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
62	retry
63	.section	.sun4v_2insn_patch, "ax"
64	.word		661b
65	nop
66	nop
67	.previous
68
69	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
70	 * instruction get nop'd out and we get here to branch
71	 * to the sun4v tlb load code.  The registers are setup
72	 * as follows:
73	 *
74	 * %g4: vaddr
75	 * %g5: PTE
76	 * %g6:	TAG
77	 *
78	 * The sun4v TLB load wants the PTE in %g3 so we fix that
79	 * up here.
80	 */
81	ba,pt		%xcc, sun4v_itlb_load
82	 mov		%g5, %g3
83
84kvmap_itlb_longpath:
85
86661:	rdpr	%pstate, %g5
87	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
88	.section .sun4v_2insn_patch, "ax"
89	.word	661b
90	SET_GL(1)
91	nop
92	.previous
93
94	rdpr	%tpc, %g5
95	ba,pt	%xcc, sparc64_realfault_common
96	 mov	FAULT_CODE_ITLB, %g4
97
98kvmap_itlb_obp:
99	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
100
101	TSB_LOCK_TAG(%g1, %g2, %g7)
102
103	TSB_WRITE(%g1, %g5, %g6)
104
105	ba,pt		%xcc, kvmap_itlb_load
106	 nop
107
108kvmap_dtlb_obp:
109	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
110
111	TSB_LOCK_TAG(%g1, %g2, %g7)
112
113	TSB_WRITE(%g1, %g5, %g6)
114
115	ba,pt		%xcc, kvmap_dtlb_load
116	 nop
117
118kvmap_linear_early:
119	sethi		%hi(kern_linear_pte_xor), %g7
120	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
121	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
122	 xor		%g2, %g4, %g5
123
124	.align		32
125kvmap_dtlb_tsb4m_load:
126	TSB_LOCK_TAG(%g1, %g2, %g7)
127	TSB_WRITE(%g1, %g5, %g6)
128	ba,pt		%xcc, kvmap_dtlb_load
129	 nop
130
131kvmap_dtlb:
132	/* %g6: TAG TARGET */
133	mov		TLB_TAG_ACCESS, %g4
134	ldxa		[%g4] ASI_DMMU, %g4
135
136	/* The kernel executes in context zero, therefore we do not
137	 * need to clear the context ID bits out of %g4 here.
138	 */
139
140	/* sun4v_dtlb_miss branches here with the missing virtual
141	 * address already loaded into %g4
142	 */
143kvmap_dtlb_4v:
144	brgez,pn	%g4, kvmap_dtlb_nonlinear
145	 nop
146
147#ifdef CONFIG_DEBUG_PAGEALLOC
148	/* Index through the base page size TSB even for linear
149	 * mappings when using page allocation debugging.
150	 */
151	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
152#else
153	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
154	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
155#endif
156	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
157	 * page table based lookup.
158	 */
159	.globl		kvmap_linear_patch
160kvmap_linear_patch:
161	ba,a,pt		%xcc, kvmap_linear_early
162
163kvmap_dtlb_vmalloc_addr:
164	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
165
166	TSB_LOCK_TAG(%g1, %g2, %g7)
167	TSB_WRITE(%g1, %g5, %g6)
168
169	/* fallthrough to TLB load */
170
171kvmap_dtlb_load:
172
173661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
174	retry
175	.section	.sun4v_2insn_patch, "ax"
176	.word		661b
177	nop
178	nop
179	.previous
180
181	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
182	 * instruction get nop'd out and we get here to branch
183	 * to the sun4v tlb load code.  The registers are setup
184	 * as follows:
185	 *
186	 * %g4: vaddr
187	 * %g5: PTE
188	 * %g6:	TAG
189	 *
190	 * The sun4v TLB load wants the PTE in %g3 so we fix that
191	 * up here.
192	 */
193	ba,pt		%xcc, sun4v_dtlb_load
194	 mov		%g5, %g3
195
196#ifdef CONFIG_SPARSEMEM_VMEMMAP
197kvmap_vmemmap:
198	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
199	ba,a,pt		%xcc, kvmap_dtlb_load
200#endif
201
202kvmap_dtlb_nonlinear:
203	/* Catch kernel NULL pointer derefs.  */
204	sethi		%hi(PAGE_SIZE), %g5
205	cmp		%g4, %g5
206	bleu,pn		%xcc, kvmap_dtlb_longpath
207	 nop
208
209#ifdef CONFIG_SPARSEMEM_VMEMMAP
210	/* Do not use the TSB for vmemmap.  */
211	sethi		%hi(VMEMMAP_BASE), %g5
212	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
213	cmp		%g4,%g5
214	bgeu,pn		%xcc, kvmap_vmemmap
215	 nop
216#endif
217
218	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
219
220kvmap_dtlb_tsbmiss:
221	sethi		%hi(MODULES_VADDR), %g5
222	cmp		%g4, %g5
223	blu,pn		%xcc, kvmap_dtlb_longpath
224	 sethi		%hi(VMALLOC_END), %g5
225	ldx		[%g5 + %lo(VMALLOC_END)], %g5
226	cmp		%g4, %g5
227	bgeu,pn		%xcc, kvmap_dtlb_longpath
228	 nop
229
230kvmap_check_obp:
231	sethi		%hi(LOW_OBP_ADDRESS), %g5
232	cmp		%g4, %g5
233	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
234	 mov		0x1, %g5
235	sllx		%g5, 32, %g5
236	cmp		%g4, %g5
237	blu,pn		%xcc, kvmap_dtlb_obp
238	 nop
239	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
240	 nop
241
242kvmap_dtlb_longpath:
243
244661:	rdpr	%pstate, %g5
245	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
246	.section .sun4v_2insn_patch, "ax"
247	.word	661b
248	SET_GL(1)
249	ldxa		[%g0] ASI_SCRATCHPAD, %g5
250	.previous
251
252	rdpr	%tl, %g3
253	cmp	%g3, 1
254
255661:	mov	TLB_TAG_ACCESS, %g4
256	ldxa	[%g4] ASI_DMMU, %g5
257	.section .sun4v_2insn_patch, "ax"
258	.word	661b
259	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
260	nop
261	.previous
262
263	/* The kernel executes in context zero, therefore we do not
264	 * need to clear the context ID bits out of %g5 here.
265	 */
266
267	be,pt	%xcc, sparc64_realfault_common
268	 mov	FAULT_CODE_DTLB, %g4
269	ba,pt	%xcc, winfix_trampoline
270	 nop
271