xref: /linux/arch/sparc/kernel/ktlb.S (revision 80d443e8876602be2c130f79c4de81e12e2a700d) 
1/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2 *
3 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
4 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
5 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
6 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
7 */
8
9#include <asm/head.h>
10#include <asm/asi.h>
11#include <asm/page.h>
12#include <asm/pgtable.h>
13#include <asm/tsb.h>
14
15	.text
16	.align		32
17
18kvmap_itlb:
19	/* g6: TAG TARGET */
20	mov		TLB_TAG_ACCESS, %g4
21	ldxa		[%g4] ASI_IMMU, %g4
22
23	/* The kernel executes in context zero, therefore we do not
24	 * need to clear the context ID bits out of %g4 here.
25	 */
26
27	/* sun4v_itlb_miss branches here with the missing virtual
28	 * address already loaded into %g4
29	 */
30kvmap_itlb_4v:
31
32	/* Catch kernel NULL pointer calls.  */
33	sethi		%hi(PAGE_SIZE), %g5
34	cmp		%g4, %g5
35	blu,pn		%xcc, kvmap_itlb_longpath
36	 nop
37
38	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
39
40kvmap_itlb_tsb_miss:
41	sethi		%hi(LOW_OBP_ADDRESS), %g5
42	cmp		%g4, %g5
43	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
44	 mov		0x1, %g5
45	sllx		%g5, 32, %g5
46	cmp		%g4, %g5
47	blu,pn		%xcc, kvmap_itlb_obp
48	 nop
49
50kvmap_itlb_vmalloc_addr:
51	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
52
53	TSB_LOCK_TAG(%g1, %g2, %g7)
54	TSB_WRITE(%g1, %g5, %g6)
55
56	/* fallthrough to TLB load */
57
58kvmap_itlb_load:
59
60661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
61	retry
62	.section	.sun4v_2insn_patch, "ax"
63	.word		661b
64	nop
65	nop
66	.previous
67
68	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
69	 * instruction get nop'd out and we get here to branch
70	 * to the sun4v tlb load code.  The registers are setup
71	 * as follows:
72	 *
73	 * %g4: vaddr
74	 * %g5: PTE
75	 * %g6:	TAG
76	 *
77	 * The sun4v TLB load wants the PTE in %g3 so we fix that
78	 * up here.
79	 */
80	ba,pt		%xcc, sun4v_itlb_load
81	 mov		%g5, %g3
82
83kvmap_itlb_longpath:
84
85661:	rdpr	%pstate, %g5
86	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
87	.section .sun4v_2insn_patch, "ax"
88	.word	661b
89	SET_GL(1)
90	nop
91	.previous
92
93	rdpr	%tpc, %g5
94	ba,pt	%xcc, sparc64_realfault_common
95	 mov	FAULT_CODE_ITLB, %g4
96
97kvmap_itlb_obp:
98	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
99
100	TSB_LOCK_TAG(%g1, %g2, %g7)
101
102	TSB_WRITE(%g1, %g5, %g6)
103
104	ba,pt		%xcc, kvmap_itlb_load
105	 nop
106
107kvmap_dtlb_obp:
108	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
109
110	TSB_LOCK_TAG(%g1, %g2, %g7)
111
112	TSB_WRITE(%g1, %g5, %g6)
113
114	ba,pt		%xcc, kvmap_dtlb_load
115	 nop
116
117kvmap_linear_early:
118	sethi		%hi(kern_linear_pte_xor), %g7
119	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
120	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
121	 xor		%g2, %g4, %g5
122
123	.align		32
124kvmap_dtlb_tsb4m_load:
125	TSB_LOCK_TAG(%g1, %g2, %g7)
126	TSB_WRITE(%g1, %g5, %g6)
127	ba,pt		%xcc, kvmap_dtlb_load
128	 nop
129
130kvmap_dtlb:
131	/* %g6: TAG TARGET */
132	mov		TLB_TAG_ACCESS, %g4
133	ldxa		[%g4] ASI_DMMU, %g4
134
135	/* The kernel executes in context zero, therefore we do not
136	 * need to clear the context ID bits out of %g4 here.
137	 */
138
139	/* sun4v_dtlb_miss branches here with the missing virtual
140	 * address already loaded into %g4
141	 */
142kvmap_dtlb_4v:
143	brgez,pn	%g4, kvmap_dtlb_nonlinear
144	 nop
145
146#ifdef CONFIG_DEBUG_PAGEALLOC
147	/* Index through the base page size TSB even for linear
148	 * mappings when using page allocation debugging.
149	 */
150	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
151#else
152	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
153	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
154#endif
155	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
156	 * page table based lookup.
157	 */
158	.globl		kvmap_linear_patch
159kvmap_linear_patch:
160	ba,a,pt		%xcc, kvmap_linear_early
161
162kvmap_dtlb_vmalloc_addr:
163	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
164
165	TSB_LOCK_TAG(%g1, %g2, %g7)
166	TSB_WRITE(%g1, %g5, %g6)
167
168	/* fallthrough to TLB load */
169
170kvmap_dtlb_load:
171
172661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
173	retry
174	.section	.sun4v_2insn_patch, "ax"
175	.word		661b
176	nop
177	nop
178	.previous
179
180	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
181	 * instruction get nop'd out and we get here to branch
182	 * to the sun4v tlb load code.  The registers are setup
183	 * as follows:
184	 *
185	 * %g4: vaddr
186	 * %g5: PTE
187	 * %g6:	TAG
188	 *
189	 * The sun4v TLB load wants the PTE in %g3 so we fix that
190	 * up here.
191	 */
192	ba,pt		%xcc, sun4v_dtlb_load
193	 mov		%g5, %g3
194
195#ifdef CONFIG_SPARSEMEM_VMEMMAP
196kvmap_vmemmap:
197	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
198	ba,a,pt		%xcc, kvmap_dtlb_load
199#endif
200
201kvmap_dtlb_nonlinear:
202	/* Catch kernel NULL pointer derefs.  */
203	sethi		%hi(PAGE_SIZE), %g5
204	cmp		%g4, %g5
205	bleu,pn		%xcc, kvmap_dtlb_longpath
206	 nop
207
208#ifdef CONFIG_SPARSEMEM_VMEMMAP
209	/* Do not use the TSB for vmemmap.  */
210	sethi		%hi(VMEMMAP_BASE), %g5
211	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
212	cmp		%g4,%g5
213	bgeu,pn		%xcc, kvmap_vmemmap
214	 nop
215#endif
216
217	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
218
219kvmap_dtlb_tsbmiss:
220	sethi		%hi(MODULES_VADDR), %g5
221	cmp		%g4, %g5
222	blu,pn		%xcc, kvmap_dtlb_longpath
223	 sethi		%hi(VMALLOC_END), %g5
224	ldx		[%g5 + %lo(VMALLOC_END)], %g5
225	cmp		%g4, %g5
226	bgeu,pn		%xcc, kvmap_dtlb_longpath
227	 nop
228
229kvmap_check_obp:
230	sethi		%hi(LOW_OBP_ADDRESS), %g5
231	cmp		%g4, %g5
232	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
233	 mov		0x1, %g5
234	sllx		%g5, 32, %g5
235	cmp		%g4, %g5
236	blu,pn		%xcc, kvmap_dtlb_obp
237	 nop
238	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
239	 nop
240
241kvmap_dtlb_longpath:
242
243661:	rdpr	%pstate, %g5
244	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
245	.section .sun4v_2insn_patch, "ax"
246	.word	661b
247	SET_GL(1)
248	ldxa		[%g0] ASI_SCRATCHPAD, %g5
249	.previous
250
251	rdpr	%tl, %g3
252	cmp	%g3, 1
253
254661:	mov	TLB_TAG_ACCESS, %g4
255	ldxa	[%g4] ASI_DMMU, %g5
256	.section .sun4v_2insn_patch, "ax"
257	.word	661b
258	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
259	nop
260	.previous
261
262	/* The kernel executes in context zero, therefore we do not
263	 * need to clear the context ID bits out of %g5 here.
264	 */
265
266	be,pt	%xcc, sparc64_realfault_common
267	 mov	FAULT_CODE_DTLB, %g4
268	ba,pt	%xcc, winfix_trampoline
269	 nop
270