xref: /linux/arch/sparc/kernel/sun4v_tlb_miss.S (revision 4949009eb8d40a441dcddcd96e101e77d31cf1b2) 
1/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
2 *
3 * Copyright (C) 2006 <davem@davemloft.net>
4 */
5
6	.text
7	.align	32
8
9	/* Load ITLB fault information into VADDR and CTX, using BASE.  */
10#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \
11	ldx	[BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \
12	ldx	[BASE + HV_FAULT_I_CTX_OFFSET], CTX;
13
14	/* Load DTLB fault information into VADDR and CTX, using BASE.  */
15#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \
16	ldx	[BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \
17	ldx	[BASE + HV_FAULT_D_CTX_OFFSET], CTX;
18
19	/* DEST = (VADDR >> 22)
20	 *
21	 * Branch to ZERO_CTX_LABEL if context is zero.
22	 */
23#define	COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \
24	srlx	VADDR, 22, DEST; \
25	brz,pn	CTX, ZERO_CTX_LABEL; \
26	 nop;
27
28	/* Create TSB pointer.  This is something like:
29	 *
30	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
31	 * tsb_base = tsb_reg & ~0x7UL;
32	 * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask);
33	 * tsb_ptr = tsb_base + (tsb_index * 16);
34	 */
35#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \
36	and	TSB_PTR, 0x7, TMP1;			\
37	mov	512, TMP2;				\
38	andn	TSB_PTR, 0x7, TSB_PTR;			\
39	sllx	TMP2, TMP1, TMP2;			\
40	srlx	VADDR, HASH_SHIFT, TMP1;		\
41	sub	TMP2, 1, TMP2;				\
42	and	TMP1, TMP2, TMP1;			\
43	sllx	TMP1, 4, TMP1;				\
44	add	TSB_PTR, TMP1, TSB_PTR;
45
46sun4v_itlb_miss:
47	/* Load MMU Miss base into %g2.  */
48	ldxa	[%g0] ASI_SCRATCHPAD, %g2
49
50	/* Load UTSB reg into %g1.  */
51	mov	SCRATCHPAD_UTSBREG1, %g1
52	ldxa	[%g1] ASI_SCRATCHPAD, %g1
53
54	LOAD_ITLB_INFO(%g2, %g4, %g5)
55	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v)
56	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
57
58	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
59	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
60	cmp	%g2, %g6
61	bne,a,pn %xcc, tsb_miss_page_table_walk
62	 mov	FAULT_CODE_ITLB, %g3
63	andcc	%g3, _PAGE_EXEC_4V, %g0
64	be,a,pn	%xcc, tsb_do_fault
65	 mov	FAULT_CODE_ITLB, %g3
66
67	/* We have a valid entry, make hypervisor call to load
68	 * I-TLB and return from trap.
69	 *
70	 * %g3:	PTE
71	 * %g4:	vaddr
72	 */
73sun4v_itlb_load:
74	ldxa	[%g0] ASI_SCRATCHPAD, %g6
75	mov	%o0, %g1		! save %o0
76	mov	%o1, %g2		! save %o1
77	mov	%o2, %g5		! save %o2
78	mov	%o3, %g7		! save %o3
79	mov	%g4, %o0		! vaddr
80	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1	! ctx
81	mov	%g3, %o2		! PTE
82	mov	HV_MMU_IMMU, %o3	! flags
83	ta	HV_MMU_MAP_ADDR_TRAP
84	brnz,pn	%o0, sun4v_itlb_error
85	 mov	%g2, %o1		! restore %o1
86	mov	%g1, %o0		! restore %o0
87	mov	%g5, %o2		! restore %o2
88	mov	%g7, %o3		! restore %o3
89
90	retry
91
92sun4v_dtlb_miss:
93	/* Load MMU Miss base into %g2.  */
94	ldxa	[%g0] ASI_SCRATCHPAD, %g2
95
96	/* Load UTSB reg into %g1.  */
97	mov	SCRATCHPAD_UTSBREG1, %g1
98	ldxa	[%g1] ASI_SCRATCHPAD, %g1
99
100	LOAD_DTLB_INFO(%g2, %g4, %g5)
101	COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v)
102	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7)
103
104	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
105	ldda	[%g1] ASI_QUAD_LDD_PHYS_4V, %g2
106	cmp	%g2, %g6
107	bne,a,pn %xcc, tsb_miss_page_table_walk
108	 mov	FAULT_CODE_DTLB, %g3
109
110	/* We have a valid entry, make hypervisor call to load
111	 * D-TLB and return from trap.
112	 *
113	 * %g3:	PTE
114	 * %g4:	vaddr
115	 */
116sun4v_dtlb_load:
117	ldxa	[%g0] ASI_SCRATCHPAD, %g6
118	mov	%o0, %g1		! save %o0
119	mov	%o1, %g2		! save %o1
120	mov	%o2, %g5		! save %o2
121	mov	%o3, %g7		! save %o3
122	mov	%g4, %o0		! vaddr
123	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1	! ctx
124	mov	%g3, %o2		! PTE
125	mov	HV_MMU_DMMU, %o3	! flags
126	ta	HV_MMU_MAP_ADDR_TRAP
127	brnz,pn	%o0, sun4v_dtlb_error
128	 mov	%g2, %o1		! restore %o1
129	mov	%g1, %o0		! restore %o0
130	mov	%g5, %o2		! restore %o2
131	mov	%g7, %o3		! restore %o3
132
133	retry
134
135sun4v_dtlb_prot:
136	SET_GL(1)
137
138	/* Load MMU Miss base into %g5.  */
139	ldxa	[%g0] ASI_SCRATCHPAD, %g5
140
141	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
142	rdpr	%tl, %g1
143	cmp	%g1, 1
144	bgu,pn	%xcc, winfix_trampoline
145	 mov	FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4
146	ba,pt	%xcc, sparc64_realfault_common
147	 nop
148
149	/* Called from trap table:
150	 * %g4:	vaddr
151	 * %g5:	context
152	 * %g6: TAG TARGET
153	 */
154sun4v_itsb_miss:
155	mov	SCRATCHPAD_UTSBREG1, %g1
156	ldxa	[%g1] ASI_SCRATCHPAD, %g1
157	brz,pn	%g5, kvmap_itlb_4v
158	 mov	FAULT_CODE_ITLB, %g3
159	ba,a,pt	%xcc, sun4v_tsb_miss_common
160
161	/* Called from trap table:
162	 * %g4:	vaddr
163	 * %g5:	context
164	 * %g6: TAG TARGET
165	 */
166sun4v_dtsb_miss:
167	mov	SCRATCHPAD_UTSBREG1, %g1
168	ldxa	[%g1] ASI_SCRATCHPAD, %g1
169	brz,pn	%g5, kvmap_dtlb_4v
170	 mov	FAULT_CODE_DTLB, %g3
171
172	/* fallthrough */
173
174sun4v_tsb_miss_common:
175	COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7)
176
177	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
178
179#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
180	mov	SCRATCHPAD_UTSBREG2, %g5
181	ldxa	[%g5] ASI_SCRATCHPAD, %g5
182	cmp	%g5, -1
183	be,pt	%xcc, 80f
184	 nop
185	COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7)
186
187	/* That clobbered %g2, reload it.  */
188	ldxa	[%g0] ASI_SCRATCHPAD, %g2
189	sub	%g2, TRAP_PER_CPU_FAULT_INFO, %g2
190
19180:	stx	%g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP]
192#endif
193
194	ba,pt	%xcc, tsb_miss_page_table_walk_sun4v_fastpath
195	 ldx	[%g2 + TRAP_PER_CPU_PGD_PADDR], %g7
196
197sun4v_itlb_error:
198	rdpr	%tl, %g1
199	cmp	%g1, 1
200	ble,pt	%icc, sun4v_bad_ra
201	 or	%g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1
202
203	sethi	%hi(sun4v_err_itlb_vaddr), %g1
204	stx	%g4, [%g1 + %lo(sun4v_err_itlb_vaddr)]
205	sethi	%hi(sun4v_err_itlb_ctx), %g1
206	ldxa	[%g0] ASI_SCRATCHPAD, %g6
207	ldx	[%g6 + HV_FAULT_I_CTX_OFFSET], %o1
208	stx	%o1, [%g1 + %lo(sun4v_err_itlb_ctx)]
209	sethi	%hi(sun4v_err_itlb_pte), %g1
210	stx	%g3, [%g1 + %lo(sun4v_err_itlb_pte)]
211	sethi	%hi(sun4v_err_itlb_error), %g1
212	stx	%o0, [%g1 + %lo(sun4v_err_itlb_error)]
213
214	sethi	%hi(1f), %g7
215	rdpr	%tl, %g4
216	ba,pt	%xcc, etraptl1
2171:	 or	%g7, %lo(1f), %g7
218	mov	%l4, %o1
219	call	sun4v_itlb_error_report
220	 add	%sp, PTREGS_OFF, %o0
221
222	/* NOTREACHED */
223
224sun4v_dtlb_error:
225	rdpr	%tl, %g1
226	cmp	%g1, 1
227	ble,pt	%icc, sun4v_bad_ra
228	 or	%g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1
229
230	sethi	%hi(sun4v_err_dtlb_vaddr), %g1
231	stx	%g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)]
232	sethi	%hi(sun4v_err_dtlb_ctx), %g1
233	ldxa	[%g0] ASI_SCRATCHPAD, %g6
234	ldx	[%g6 + HV_FAULT_D_CTX_OFFSET], %o1
235	stx	%o1, [%g1 + %lo(sun4v_err_dtlb_ctx)]
236	sethi	%hi(sun4v_err_dtlb_pte), %g1
237	stx	%g3, [%g1 + %lo(sun4v_err_dtlb_pte)]
238	sethi	%hi(sun4v_err_dtlb_error), %g1
239	stx	%o0, [%g1 + %lo(sun4v_err_dtlb_error)]
240
241	sethi	%hi(1f), %g7
242	rdpr	%tl, %g4
243	ba,pt	%xcc, etraptl1
2441:	 or	%g7, %lo(1f), %g7
245	mov	%l4, %o1
246	call	sun4v_dtlb_error_report
247	 add	%sp, PTREGS_OFF, %o0
248
249	/* NOTREACHED */
250
251sun4v_bad_ra:
252	or	%g0, %g4, %g5
253	ba,pt	%xcc, sparc64_realfault_common
254	 or	%g1, %g0, %g4
255
256	/* NOTREACHED */
257
258	/* Instruction Access Exception, tl0. */
259sun4v_iacc:
260	ldxa	[%g0] ASI_SCRATCHPAD, %g2
261	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
262	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
263	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
264	sllx	%g3, 16, %g3
265	or	%g5, %g3, %g5
266	ba,pt	%xcc, etrap
267	 rd	%pc, %g7
268	mov	%l4, %o1
269	mov	%l5, %o2
270	call	sun4v_insn_access_exception
271	 add	%sp, PTREGS_OFF, %o0
272	ba,a,pt	%xcc, rtrap
273
274	/* Instruction Access Exception, tl1. */
275sun4v_iacc_tl1:
276	ldxa	[%g0] ASI_SCRATCHPAD, %g2
277	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
278	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
279	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
280	sllx	%g3, 16, %g3
281	or	%g5, %g3, %g5
282	ba,pt	%xcc, etraptl1
283	 rd	%pc, %g7
284	mov	%l4, %o1
285	mov	%l5, %o2
286	call	sun4v_insn_access_exception_tl1
287	 add	%sp, PTREGS_OFF, %o0
288	ba,a,pt	%xcc, rtrap
289
290	/* Data Access Exception, tl0. */
291sun4v_dacc:
292	ldxa	[%g0] ASI_SCRATCHPAD, %g2
293	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
294	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
295	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
296	sllx	%g3, 16, %g3
297	or	%g5, %g3, %g5
298	ba,pt	%xcc, etrap
299	 rd	%pc, %g7
300	mov	%l4, %o1
301	mov	%l5, %o2
302	call	sun4v_data_access_exception
303	 add	%sp, PTREGS_OFF, %o0
304	ba,a,pt	%xcc, rtrap
305
306	/* Data Access Exception, tl1. */
307sun4v_dacc_tl1:
308	ldxa	[%g0] ASI_SCRATCHPAD, %g2
309	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
310	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
311	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
312	sllx	%g3, 16, %g3
313	or	%g5, %g3, %g5
314	ba,pt	%xcc, etraptl1
315	 rd	%pc, %g7
316	mov	%l4, %o1
317	mov	%l5, %o2
318	call	sun4v_data_access_exception_tl1
319	 add	%sp, PTREGS_OFF, %o0
320	ba,a,pt	%xcc, rtrap
321
322	/* Memory Address Unaligned.  */
323sun4v_mna:
324	/* Window fixup? */
325	rdpr	%tl, %g2
326	cmp	%g2, 1
327	ble,pt	%icc, 1f
328	 nop
329
330	SET_GL(1)
331	ldxa	[%g0] ASI_SCRATCHPAD, %g2
332	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
333	mov	HV_FAULT_TYPE_UNALIGNED, %g3
334	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g4
335	sllx	%g3, 16, %g3
336	or	%g4, %g3, %g4
337	ba,pt	%xcc, winfix_mna
338	 rdpr	%tpc, %g3
339	/* not reached */
340
3411:	ldxa	[%g0] ASI_SCRATCHPAD, %g2
342	mov	HV_FAULT_TYPE_UNALIGNED, %g3
343	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
344	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
345	sllx	%g3, 16, %g3
346	or	%g5, %g3, %g5
347
348	ba,pt	%xcc, etrap
349	 rd	%pc, %g7
350	mov	%l4, %o1
351	mov	%l5, %o2
352	call	sun4v_do_mna
353	 add	%sp, PTREGS_OFF, %o0
354	ba,a,pt	%xcc, rtrap
355
356	/* Privileged Action.  */
357sun4v_privact:
358	ba,pt	%xcc, etrap
359	 rd	%pc, %g7
360	call	do_privact
361	 add	%sp, PTREGS_OFF, %o0
362	ba,a,pt	%xcc, rtrap
363
364	/* Unaligned ldd float, tl0. */
365sun4v_lddfmna:
366	ldxa	[%g0] ASI_SCRATCHPAD, %g2
367	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
368	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
369	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
370	sllx	%g3, 16, %g3
371	or	%g5, %g3, %g5
372	ba,pt	%xcc, etrap
373	 rd	%pc, %g7
374	mov	%l4, %o1
375	mov	%l5, %o2
376	call	handle_lddfmna
377	 add	%sp, PTREGS_OFF, %o0
378	ba,a,pt	%xcc, rtrap
379
380	/* Unaligned std float, tl0. */
381sun4v_stdfmna:
382	ldxa	[%g0] ASI_SCRATCHPAD, %g2
383	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
384	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
385	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
386	sllx	%g3, 16, %g3
387	or	%g5, %g3, %g5
388	ba,pt	%xcc, etrap
389	 rd	%pc, %g7
390	mov	%l4, %o1
391	mov	%l5, %o2
392	call	handle_stdfmna
393	 add	%sp, PTREGS_OFF, %o0
394	ba,a,pt	%xcc, rtrap
395
396#define BRANCH_ALWAYS	0x10680000
397#define NOP		0x01000000
398#define SUN4V_DO_PATCH(OLD, NEW)	\
399	sethi	%hi(NEW), %g1; \
400	or	%g1, %lo(NEW), %g1; \
401	sethi	%hi(OLD), %g2; \
402	or	%g2, %lo(OLD), %g2; \
403	sub	%g1, %g2, %g1; \
404	sethi	%hi(BRANCH_ALWAYS), %g3; \
405	sll	%g1, 11, %g1; \
406	srl	%g1, 11 + 2, %g1; \
407	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
408	or	%g3, %g1, %g3; \
409	stw	%g3, [%g2]; \
410	sethi	%hi(NOP), %g3; \
411	or	%g3, %lo(NOP), %g3; \
412	stw	%g3, [%g2 + 0x4]; \
413	flush	%g2;
414
415	.globl	sun4v_patch_tlb_handlers
416	.type	sun4v_patch_tlb_handlers,#function
417sun4v_patch_tlb_handlers:
418	SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
419	SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
420	SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
421	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
422	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
423	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
424	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
425	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
426	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
427	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
428	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
429	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
430	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
431	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
432	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
433	retl
434	 nop
435	.size	sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
436