xref: /freebsd/sys/powerpc/aim/trap_subr64.S (revision b4e38a41f584ad4391c04b8cfec81f46176b18b0)
1/* $FreeBSD$ */
2/* $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $	*/
3
4/*-
5 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
6 * Copyright (C) 1995, 1996 TooLs GmbH.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 *    must display the following acknowledgement:
19 *	This product includes software developed by TooLs GmbH.
20 * 4. The name of TooLs GmbH may not be used to endorse or promote products
21 *    derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
28 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
29 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
31 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
32 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35/*
36 * NOTICE: This is not a standalone file.  to use it, #include it in
37 * your port's locore.S, like so:
38 *
39 *	#include <powerpc/aim/trap_subr.S>
40 */
41
42/* Locate the per-CPU data structure */
43#define GET_CPUINFO(r)  \
44        mfsprg0  r
45#define GET_TOCBASE(r)  \
46	lis	r,DMAP_BASE_ADDRESS@highesta;	/* To real-mode alias/dmap */ \
47	sldi	r,r,32;							\
48	ori	r,r,TRAP_TOCBASE;	/* Magic address for TOC */	\
49	ld	r,0(r)
50
51/*
52 * Restore SRs for a pmap
53 *
54 * Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
55 */
56
57/*
58 * User SRs are loaded through a pointer to the current pmap.
59 */
60restore_usersrs:
61	GET_CPUINFO(%r28)
62	ld	%r28,PC_USERSLB(%r28)
63	cmpdi	%r28, 0			/* If user SLB pointer NULL, exit */
64	beqlr
65
66	li	%r29, 0			/* Set the counter to zero */
67
68	slbia
69	slbmfee	%r31,%r29
70	clrrdi	%r31,%r31,28
71	slbie	%r31
721:	ld	%r31, 0(%r28)		/* Load SLB entry pointer */
73	cmpdi	%r31, 0			/* If NULL, stop */
74	beqlr
75
76	ld	%r30, 0(%r31)		/* Load SLBV */
77	ld	%r31, 8(%r31)		/* Load SLBE */
78	or	%r31, %r31, %r29	/*  Set SLBE slot */
79	slbmte	%r30, %r31		/* Install SLB entry */
80
81	addi	%r28, %r28, 8		/* Advance pointer */
82	addi	%r29, %r29, 1
83	b	1b			/* Repeat */
84
85/*
86 * Kernel SRs are loaded directly from the PCPU fields
87 */
88restore_kernsrs:
89	GET_CPUINFO(%r28)
90	lwz	%r29, PC_FLAGS(%r28)
91	mtcr	%r29
92	btlr	0
93	addi	%r28,%r28,PC_KERNSLB
94	ld	%r29,16(%r28)		/* One past USER_SLB_SLOT */
95	cmpdi	%r29,0
96	beqlr				/* If first kernel entry is invalid,
97					 * SLBs not in use, so exit early */
98
99	/* Otherwise, set up SLBs */
100	li	%r29, 0			/* Set the counter to zero */
101
102	slbia
103	slbmfee	%r31,%r29
104	clrrdi	%r31,%r31,28
105	slbie	%r31
1061:	cmpdi	%r29, USER_SLB_SLOT	/* Skip the user slot */
107	beq-	2f
108
109	ld	%r31, 8(%r28)		/* Load SLBE */
110	cmpdi	%r31, 0			/* If SLBE is not valid, stop */
111	beqlr
112	ld	%r30, 0(%r28)		/* Load SLBV  */
113	slbmte	%r30, %r31		/* Install SLB entry */
114
1152:	addi	%r28, %r28, 16		/* Advance pointer */
116	addi	%r29, %r29, 1
117	cmpdi	%r29, 64		/* Repeat if we are not at the end */
118	blt	1b
119	blr
120
121/*
122 * FRAME_SETUP assumes:
123 *	SPRG1		SP (1)
124 * 	SPRG3		trap type
125 *	savearea	r27-r31,DAR,DSISR   (DAR & DSISR only for DSI traps)
126 *	r28		LR
127 *	r29		CR
128 *	r30		scratch
129 *	r31		scratch
130 *	r1		kernel stack
131 *	SRR0/1		as at start of trap
132 *
133 * NOTE: SPRG1 is never used while the MMU is on, making it safe to reuse
134 * in any real-mode fault handler, including those handling double faults.
135 */
136#define	FRAME_SETUP(savearea)						\
137/* Have to enable translation to allow access of kernel stack: */	\
138	GET_CPUINFO(%r31);						\
139	mfsrr0	%r30;							\
140	std	%r30,(savearea+CPUSAVE_SRR0)(%r31);	/* save SRR0 */	\
141	mfsrr1	%r30;							\
142	std	%r30,(savearea+CPUSAVE_SRR1)(%r31);	/* save SRR1 */	\
143	mfsprg1	%r31;			/* get saved SP (clears SPRG1) */ \
144	mfmsr	%r30;							\
145	ori	%r30,%r30,(PSL_DR|PSL_IR|PSL_RI)@l; /* relocation on */	\
146	mtmsr	%r30;			/* stack can now be accessed */	\
147	isync;								\
148	stdu	%r31,-(FRAMELEN+288)(%r1); /* save it in the callframe */ \
149	std	%r0, FRAME_0+48(%r1);	/* save r0 in the trapframe */	\
150	std	%r31,FRAME_1+48(%r1);	/* save SP   "      "       */	\
151	std	%r2, FRAME_2+48(%r1);	/* save r2   "      "       */	\
152	std	%r28,FRAME_LR+48(%r1);	/* save LR   "      "       */	\
153	std	%r29,FRAME_CR+48(%r1);	/* save CR   "      "       */	\
154	GET_CPUINFO(%r2);						\
155	ld	%r27,(savearea+CPUSAVE_R27)(%r2); /* get saved r27 */	\
156	ld	%r28,(savearea+CPUSAVE_R28)(%r2); /* get saved r28 */	\
157	ld	%r29,(savearea+CPUSAVE_R29)(%r2); /* get saved r29 */	\
158	ld	%r30,(savearea+CPUSAVE_R30)(%r2); /* get saved r30 */	\
159	ld	%r31,(savearea+CPUSAVE_R31)(%r2); /* get saved r31 */	\
160	std	%r3,  FRAME_3+48(%r1);	/* save r3-r31 */		\
161	std	%r4,  FRAME_4+48(%r1);					\
162	std	%r5,  FRAME_5+48(%r1);					\
163	std	%r6,  FRAME_6+48(%r1);					\
164	std	%r7,  FRAME_7+48(%r1);					\
165	std	%r8,  FRAME_8+48(%r1);					\
166	std	%r9,  FRAME_9+48(%r1);					\
167	std	%r10, FRAME_10+48(%r1);					\
168	std	%r11, FRAME_11+48(%r1);					\
169	std	%r12, FRAME_12+48(%r1);					\
170	std	%r13, FRAME_13+48(%r1);					\
171	std	%r14, FRAME_14+48(%r1);					\
172	std	%r15, FRAME_15+48(%r1);					\
173	std	%r16, FRAME_16+48(%r1);					\
174	std	%r17, FRAME_17+48(%r1);					\
175	std	%r18, FRAME_18+48(%r1);					\
176	std	%r19, FRAME_19+48(%r1);					\
177	std	%r20, FRAME_20+48(%r1);					\
178	std	%r21, FRAME_21+48(%r1);					\
179	std	%r22, FRAME_22+48(%r1);					\
180	std	%r23, FRAME_23+48(%r1);					\
181	std	%r24, FRAME_24+48(%r1);					\
182	std	%r25, FRAME_25+48(%r1);					\
183	std	%r26, FRAME_26+48(%r1);					\
184	std	%r27, FRAME_27+48(%r1);					\
185	std	%r28, FRAME_28+48(%r1);					\
186	std	%r29, FRAME_29+48(%r1);					\
187	std	%r30, FRAME_30+48(%r1);					\
188	std	%r31, FRAME_31+48(%r1);					\
189	ld	%r28,(savearea+CPUSAVE_AIM_DAR)(%r2);  /* saved DAR */	\
190	ld	%r29,(savearea+CPUSAVE_AIM_DSISR)(%r2);/* saved DSISR */\
191	ld	%r30,(savearea+CPUSAVE_SRR0)(%r2); /* saved SRR0 */	\
192	ld	%r31,(savearea+CPUSAVE_SRR1)(%r2); /* saved SRR1 */	\
193	mfxer	%r3;							\
194	mfctr	%r4;							\
195	mfsprg3	%r5;							\
196	std	%r3, FRAME_XER+48(1);	/* save xer/ctr/exc */		\
197	std	%r4, FRAME_CTR+48(1);					\
198	std	%r5, FRAME_EXC+48(1);					\
199	std	%r28,FRAME_AIM_DAR+48(1);				\
200	std	%r29,FRAME_AIM_DSISR+48(1); /* save dsisr/srr0/srr1 */	\
201	std	%r30,FRAME_SRR0+48(1);					\
202	std	%r31,FRAME_SRR1+48(1);					\
203	ld	%r13,PC_CURTHREAD(%r2)	/* set kernel curthread */
204
205#define	FRAME_LEAVE(savearea)						\
206/* Disable exceptions: */						\
207	mfmsr	%r2;							\
208	andi.	%r2,%r2,~PSL_EE@l;					\
209	mtmsr	%r2;							\
210	isync;								\
211/* Now restore regs: */							\
212	ld	%r2,FRAME_SRR0+48(%r1);					\
213	ld	%r3,FRAME_SRR1+48(%r1);					\
214	ld	%r4,FRAME_CTR+48(%r1);					\
215	ld	%r5,FRAME_XER+48(%r1);					\
216	ld	%r6,FRAME_LR+48(%r1);					\
217	GET_CPUINFO(%r7);						\
218	std	%r2,(savearea+CPUSAVE_SRR0)(%r7); /* save SRR0 */	\
219	std	%r3,(savearea+CPUSAVE_SRR1)(%r7); /* save SRR1 */	\
220	ld	%r7,FRAME_CR+48(%r1);					\
221	mtctr	%r4;							\
222	mtxer	%r5;							\
223	mtlr	%r6;							\
224	mtsprg2	%r7;			/* save cr */			\
225	ld	%r31,FRAME_31+48(%r1);   /* restore r0-31 */		\
226	ld	%r30,FRAME_30+48(%r1);					\
227	ld	%r29,FRAME_29+48(%r1);					\
228	ld	%r28,FRAME_28+48(%r1);					\
229	ld	%r27,FRAME_27+48(%r1);					\
230	ld	%r26,FRAME_26+48(%r1);					\
231	ld	%r25,FRAME_25+48(%r1);					\
232	ld	%r24,FRAME_24+48(%r1);					\
233	ld	%r23,FRAME_23+48(%r1);					\
234	ld	%r22,FRAME_22+48(%r1);					\
235	ld	%r21,FRAME_21+48(%r1);					\
236	ld	%r20,FRAME_20+48(%r1);					\
237	ld	%r19,FRAME_19+48(%r1);					\
238	ld	%r18,FRAME_18+48(%r1);					\
239	ld	%r17,FRAME_17+48(%r1);					\
240	ld	%r16,FRAME_16+48(%r1);					\
241	ld	%r15,FRAME_15+48(%r1);					\
242	ld	%r14,FRAME_14+48(%r1);					\
243	ld	%r13,FRAME_13+48(%r1);					\
244	ld	%r12,FRAME_12+48(%r1);					\
245	ld	%r11,FRAME_11+48(%r1);					\
246	ld	%r10,FRAME_10+48(%r1);					\
247	ld	%r9, FRAME_9+48(%r1);					\
248	ld	%r8, FRAME_8+48(%r1);					\
249	ld	%r7, FRAME_7+48(%r1);					\
250	ld	%r6, FRAME_6+48(%r1);					\
251	ld	%r5, FRAME_5+48(%r1);					\
252	ld	%r4, FRAME_4+48(%r1);					\
253	ld	%r3, FRAME_3+48(%r1);					\
254	ld	%r2, FRAME_2+48(%r1);					\
255	ld	%r0, FRAME_0+48(%r1);					\
256	ld	%r1, FRAME_1+48(%r1);					\
257/* Can't touch %r1 from here on */					\
258	mtsprg3	%r3;			/* save r3 */			\
259/* Disable translation, machine check and recoverability: */		\
260	mfmsr	%r3;							\
261	andi.	%r3,%r3,~(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;		\
262	mtmsr	%r3;							\
263	isync;								\
264/* Decide whether we return to user mode: */				\
265	GET_CPUINFO(%r3);						\
266	ld	%r3,(savearea+CPUSAVE_SRR1)(%r3);			\
267	mtcr	%r3;							\
268	bf	17,1f;			/* branch if PSL_PR is false */	\
269/* Restore user SRs */							\
270	GET_CPUINFO(%r3);						\
271	std	%r27,(savearea+CPUSAVE_R27)(%r3);			\
272	std	%r28,(savearea+CPUSAVE_R28)(%r3);			\
273	std	%r29,(savearea+CPUSAVE_R29)(%r3);			\
274	std	%r30,(savearea+CPUSAVE_R30)(%r3);			\
275	std	%r31,(savearea+CPUSAVE_R31)(%r3);			\
276	lwz	%r28,PC_FLAGS(%r3);					\
277	mtcr	%r28;							\
278	bt	0, 0f;	/* Check to skip restoring SRs. */		\
279	mflr	%r27;			/* preserve LR */		\
280	bl	restore_usersrs;	/* uses r28-r31 */		\
281	mtlr	%r27;							\
2820:									\
283	ld	%r31,(savearea+CPUSAVE_R31)(%r3);			\
284	ld	%r30,(savearea+CPUSAVE_R30)(%r3);			\
285	ld	%r29,(savearea+CPUSAVE_R29)(%r3);			\
286	ld	%r28,(savearea+CPUSAVE_R28)(%r3);			\
287	ld	%r27,(savearea+CPUSAVE_R27)(%r3);			\
2881:	mfsprg2	%r3;			/* restore cr */		\
289	mtcr	%r3;							\
290	GET_CPUINFO(%r3);						\
291	ld	%r3,(savearea+CPUSAVE_SRR0)(%r3); /* restore srr0 */	\
292	mtsrr0	%r3;							\
293	GET_CPUINFO(%r3);						\
294	ld	%r3,(savearea+CPUSAVE_SRR1)(%r3); /* restore srr1 */	\
295	mtsrr1	%r3;							\
296	mfsprg3	%r3			/* restore r3 */
297
298#ifdef KDTRACE_HOOKS
299	.data
300	.globl	dtrace_invop_calltrap_addr
301	.align	8
302	.type	dtrace_invop_calltrap_addr, @object
303        .size	dtrace_invop_calltrap_addr, 8
304dtrace_invop_calltrap_addr:
305	.word	0
306	.word	0
307
308	.text
309#endif
310
311/*
312 * Processor reset exception handler. These are typically
313 * the first instructions the processor executes after a
314 * software reset. We do this in two bits so that we are
315 * not still hanging around in the trap handling region
316 * once the MMU is turned on.
317 */
318	.globl	CNAME(rstcode), CNAME(rstcodeend), CNAME(cpu_reset_handler)
319	.globl	CNAME(cpu_wakeup_handler)
320	.p2align 3
321CNAME(rstcode):
322	/*
323	 * Check if this is software reset or
324	 * processor is waking up from power saving mode
325	 * It is software reset when 46:47 = 0b00
326	 */
327	/* 0x00 */
328	ld	%r2,TRAP_GENTRAP(0)	/* Real-mode &generictrap */
329	mfsrr1	%r9			/* Load SRR1 into r9 */
330	andis.	%r9,%r9,0x3		/* Logic AND with 46:47 bits */
331
332	beq	2f			/* Branch if software reset */
333	/* 0x10 */
334	/* Reset was wakeup */
335	addi	%r9,%r2,(cpu_wakeup_handler-generictrap)
336	b	1f			/* Was power save, do the wakeup */
337
338	/* Reset was software reset */
339	/* Explicitly set MSR[SF] */
3402:	mfmsr	%r9
341	li	%r8,1
342	/* 0x20 */
343	insrdi	%r9,%r8,1,0
344	mtmsrd	%r9
345	isync
346
347	addi	%r9,%r2,(cpu_reset_handler-generictrap)
348
349	/* 0x30 */
3501:	mtlr	%r9
351	blr				/* Branch to either cpu_reset_handler
352					 * or cpu_wakeup_handler.
353					 */
354CNAME(rstcodeend):
355
356cpu_reset_handler:
357	GET_TOCBASE(%r2)
358
359	addis	%r1,%r2,TOC_REF(tmpstk)@ha
360	ld	%r1,TOC_REF(tmpstk)@l(%r1)	/* get new SP */
361	addi	%r1,%r1,(TMPSTKSZ-48)
362
363	bl	CNAME(cpudep_ap_early_bootstrap) /* Set PCPU */
364	nop
365	lis	%r3,1@l
366	bl	CNAME(pmap_cpu_bootstrap)	/* Turn on virtual memory */
367	nop
368	bl	CNAME(cpudep_ap_bootstrap)	/* Set up PCPU and stack */
369	nop
370	mr	%r1,%r3				/* Use new stack */
371	bl	CNAME(cpudep_ap_setup)
372	nop
373	GET_CPUINFO(%r5)
374	ld	%r3,(PC_RESTORE)(%r5)
375	cmpldi	%cr0,%r3,0
376	beq	%cr0,2f
377	nop
378	li	%r4,1
379	bl	CNAME(longjmp)
380	nop
3812:
382#ifdef SMP
383	bl	CNAME(machdep_ap_bootstrap)	/* And away! */
384	nop
385#endif
386
387	/* Should not be reached */
3889:
389	b	9b
390
391cpu_wakeup_handler:
392	GET_TOCBASE(%r2)
393
394	/* Check for false wake up due to badly SRR1 set (eg. by OPAL) */
395	addis	%r3,%r2,TOC_REF(can_wakeup)@ha
396	ld	%r3,TOC_REF(can_wakeup)@l(%r3)
397	ld	%r3,0(%r3)
398	cmpdi	%r3,0
399	beq	cpu_reset_handler
400
401	/* Turn on MMU after return from interrupt */
402	mfsrr1	%r3
403	ori	%r3,%r3,(PSL_IR | PSL_DR)
404	mtsrr1	%r3
405
406	/* Turn on MMU (needed to access PCB) */
407	mfmsr	%r3
408	ori	%r3,%r3,(PSL_IR | PSL_DR)
409	mtmsr	%r3
410	isync
411
412	mfsprg0	%r3
413
414	ld	%r3,PC_CURTHREAD(%r3)	/* Get current thread */
415	ld	%r3,TD_PCB(%r3)		/* Get PCB of current thread */
416	ld	%r12,PCB_CONTEXT(%r3)	/* Load the non-volatile GP regs. */
417	ld	%r13,PCB_CONTEXT+1*8(%r3)
418	ld	%r14,PCB_CONTEXT+2*8(%r3)
419	ld	%r15,PCB_CONTEXT+3*8(%r3)
420	ld	%r16,PCB_CONTEXT+4*8(%r3)
421	ld	%r17,PCB_CONTEXT+5*8(%r3)
422	ld	%r18,PCB_CONTEXT+6*8(%r3)
423	ld	%r19,PCB_CONTEXT+7*8(%r3)
424	ld	%r20,PCB_CONTEXT+8*8(%r3)
425	ld	%r21,PCB_CONTEXT+9*8(%r3)
426	ld	%r22,PCB_CONTEXT+10*8(%r3)
427	ld	%r23,PCB_CONTEXT+11*8(%r3)
428	ld	%r24,PCB_CONTEXT+12*8(%r3)
429	ld	%r25,PCB_CONTEXT+13*8(%r3)
430	ld	%r26,PCB_CONTEXT+14*8(%r3)
431	ld	%r27,PCB_CONTEXT+15*8(%r3)
432	ld	%r28,PCB_CONTEXT+16*8(%r3)
433	ld	%r29,PCB_CONTEXT+17*8(%r3)
434	ld	%r30,PCB_CONTEXT+18*8(%r3)
435	ld	%r31,PCB_CONTEXT+19*8(%r3)
436	ld	%r5,PCB_CR(%r3)		/* Load the condition register */
437	mtcr	%r5
438	ld	%r5,PCB_LR(%r3)		/* Load the link register */
439	mtsrr0	%r5
440	ld	%r1,PCB_SP(%r3)		/* Load the stack pointer */
441	ld	%r2,PCB_TOC(%r3)	/* Load the TOC pointer */
442
443	rfid
444
445/*
446 * This code gets copied to all the trap vectors
447 * (except ISI/DSI, ALI, and the interrupts). Has to fit in 8 instructions!
448 */
449
450	.globl	CNAME(trapcode),CNAME(trapcodeend)
451	.p2align 3
452CNAME(trapcode):
453	mtsprg1	%r1			/* save SP */
454	mflr	%r1			/* Save the old LR in r1 */
455	mtsprg2 %r1			/* And then in SPRG2 */
456	ld	%r1,TRAP_ENTRY(0)
457	mtlr	%r1
458	li	%r1, 0xe0		/* How to get the vector from LR */
459	blrl				/* Branch to generictrap */
460CNAME(trapcodeend):
461
462/* Same thing for traps setting HSRR0/HSRR1 */
463	.globl	CNAME(hypertrapcode),CNAME(hypertrapcodeend)
464	.p2align 3
465CNAME(hypertrapcode):
466	mtsprg1	%r1			/* save SP */
467	mflr	%r1			/* Save the old LR in r1 */
468	mtsprg2 %r1			/* And then in SPRG2 */
469	ld	%r1,TRAP_GENTRAP(0)
470	addi	%r1,%r1,(generichypertrap-generictrap)
471	mtlr	%r1
472	li	%r1, 0xe0		/* How to get the vector from LR */
473	blrl				/* Branch to generichypertrap */
474CNAME(hypertrapcodeend):
475
476/*
477 * For SLB misses: do special things for the kernel
478 *
479 * Note: SPRG1 is always safe to overwrite any time the MMU was on, which is
480 * the only time this can be called.
481 */
482	.globl	CNAME(slbtrap),CNAME(slbtrapend)
483	.p2align 3
484CNAME(slbtrap):
485	/* 0x00 */
486	mtsprg1	%r1			/* save SP */
487	GET_CPUINFO(%r1)
488	std	%r2,(PC_SLBSAVE+16)(%r1)	/* save r2 */
489	mfcr	%r2
490	/* 0x10 */
491	std	%r2,(PC_SLBSAVE+104)(%r1)	/* save CR */
492	mfsrr1	%r2			/* test kernel mode */
493	mtcr	%r2
494	bf	17,2f			/* branch if PSL_PR is false */
495	/* 0x20 */
496	/* User mode */
497	ld	%r2,(PC_SLBSAVE+104)(%r1)
498	mtcr	%r2				/* restore CR */
499	ld	%r2,(PC_SLBSAVE+16)(%r1) 	/* restore r2 */
500	mflr	%r1
501	/* 0x30 */
502	mtsprg2 %r1				/* save LR in SPRG2 */
503	ld	%r1,TRAP_ENTRY(0)		/* real-mode &generictrap */
504	mtlr	%r1
505	li	%r1, 0x80		/* How to get the vector from LR */
506	/* 0x40 */
507	blrl				/* Branch to generictrap */
5082:	mflr	%r2			/* Save the old LR in r2 */
509	/* Kernel mode */
510	ld	%r1,TRAP_GENTRAP(0)		/* Real-mode &generictrap */
511	addi    %r1,%r1,(kern_slbtrap-generictrap)
512	/* 0x50 */
513	mtlr	%r1
514	GET_CPUINFO(%r1)
515	blrl					/* Branch to kern_slbtrap */
516/* must fit in 128 bytes! */
517CNAME(slbtrapend):
518
519/*
520 * On entry:
521 * SPRG1: SP
522 * r1: pcpu
523 * r2: LR
524 * LR: branch address in trap region
525 */
526kern_slbtrap:
527	std	%r2,(PC_SLBSAVE+136)(%r1) /* old LR */
528	std	%r3,(PC_SLBSAVE+24)(%r1) /* save R3 */
529
530	/* Check if this needs to be handled as a regular trap (userseg miss) */
531	mflr	%r2
532	andi.	%r2,%r2,0xff80
533	cmpwi	%r2,EXC_DSE
534	bne	1f
535	mfdar	%r2
536	b	2f
5371:	mfsrr0	%r2
5382:	/* r2 now contains the fault address */
539	lis	%r3,SEGMENT_MASK@highesta
540	ori	%r3,%r3,SEGMENT_MASK@highera
541	sldi	%r3,%r3,32
542	oris	%r3,%r3,SEGMENT_MASK@ha
543	ori	%r3,%r3,SEGMENT_MASK@l
544	and	%r2,%r2,%r3	/* R2 = segment base address */
545	lis	%r3,USER_ADDR@highesta
546	ori	%r3,%r3,USER_ADDR@highera
547	sldi	%r3,%r3,32
548	oris	%r3,%r3,USER_ADDR@ha
549	ori	%r3,%r3,USER_ADDR@l
550	cmpd	%r2,%r3		/* Compare fault base to USER_ADDR */
551	bne	3f
552
553	/* User seg miss, handle as a regular trap */
554	ld	%r2,(PC_SLBSAVE+104)(%r1) /* Restore CR */
555	mtcr	%r2
556	ld	%r2,(PC_SLBSAVE+16)(%r1) /* Restore R2,R3 */
557	ld	%r3,(PC_SLBSAVE+24)(%r1)
558	ld	%r1,(PC_SLBSAVE+136)(%r1) /* Save the old LR in r1 */
559	mtsprg2 %r1			/* And then in SPRG2 */
560	li	%r1, 0x80		/* How to get the vector from LR */
561	b	generictrap		/* Retain old LR using b */
562
5633:	/* Real kernel SLB miss */
564	std	%r0,(PC_SLBSAVE+0)(%r1)	/* free all volatile regs */
565	mfsprg1	%r2			/* Old R1 */
566	std	%r2,(PC_SLBSAVE+8)(%r1)
567	/* R2,R3 already saved */
568	std	%r4,(PC_SLBSAVE+32)(%r1)
569	std	%r5,(PC_SLBSAVE+40)(%r1)
570	std	%r6,(PC_SLBSAVE+48)(%r1)
571	std	%r7,(PC_SLBSAVE+56)(%r1)
572	std	%r8,(PC_SLBSAVE+64)(%r1)
573	std	%r9,(PC_SLBSAVE+72)(%r1)
574	std	%r10,(PC_SLBSAVE+80)(%r1)
575	std	%r11,(PC_SLBSAVE+88)(%r1)
576	std	%r12,(PC_SLBSAVE+96)(%r1)
577	/* CR already saved */
578	mfxer	%r2			/* save XER */
579	std	%r2,(PC_SLBSAVE+112)(%r1)
580	mflr	%r2			/* save LR (SP already saved) */
581	std	%r2,(PC_SLBSAVE+120)(%r1)
582	mfctr	%r2			/* save CTR */
583	std	%r2,(PC_SLBSAVE+128)(%r1)
584
585	/* Call handler */
586	addi	%r1,%r1,PC_SLBSTACK-48+1024
587	li	%r2,~15
588	and	%r1,%r1,%r2
589	GET_TOCBASE(%r2)
590	mflr	%r3
591	andi.	%r3,%r3,0xff80
592	mfdar	%r4
593	mfsrr0	%r5
594	bl	handle_kernel_slb_spill
595	nop
596
597	/* Save r28-31, restore r4-r12 */
598	GET_CPUINFO(%r1)
599	ld	%r4,(PC_SLBSAVE+32)(%r1)
600	ld	%r5,(PC_SLBSAVE+40)(%r1)
601	ld	%r6,(PC_SLBSAVE+48)(%r1)
602	ld	%r7,(PC_SLBSAVE+56)(%r1)
603	ld	%r8,(PC_SLBSAVE+64)(%r1)
604	ld	%r9,(PC_SLBSAVE+72)(%r1)
605	ld	%r10,(PC_SLBSAVE+80)(%r1)
606	ld	%r11,(PC_SLBSAVE+88)(%r1)
607	ld	%r12,(PC_SLBSAVE+96)(%r1)
608	std	%r28,(PC_SLBSAVE+64)(%r1)
609	std	%r29,(PC_SLBSAVE+72)(%r1)
610	std	%r30,(PC_SLBSAVE+80)(%r1)
611	std	%r31,(PC_SLBSAVE+88)(%r1)
612
613	/* Restore kernel mapping */
614	bl	restore_kernsrs
615
616	/* Restore remaining registers */
617	ld	%r28,(PC_SLBSAVE+64)(%r1)
618	ld	%r29,(PC_SLBSAVE+72)(%r1)
619	ld	%r30,(PC_SLBSAVE+80)(%r1)
620	ld	%r31,(PC_SLBSAVE+88)(%r1)
621
622	ld	%r2,(PC_SLBSAVE+104)(%r1)
623	mtcr	%r2
624	ld	%r2,(PC_SLBSAVE+112)(%r1)
625	mtxer	%r2
626	ld	%r2,(PC_SLBSAVE+120)(%r1)
627	mtlr	%r2
628	ld	%r2,(PC_SLBSAVE+128)(%r1)
629	mtctr	%r2
630	ld	%r2,(PC_SLBSAVE+136)(%r1)
631	mtlr	%r2
632
633	/* Restore r0-r3 */
634	ld	%r0,(PC_SLBSAVE+0)(%r1)
635	ld	%r2,(PC_SLBSAVE+16)(%r1)
636	ld	%r3,(PC_SLBSAVE+24)(%r1)
637	mfsprg1	%r1
638
639	/* Back to whatever we were doing */
640	rfid
641
642/*
643 * For ALI: has to save DSISR and DAR
644 */
645	.globl	CNAME(alitrap),CNAME(aliend)
646CNAME(alitrap):
647	mtsprg1	%r1			/* save SP */
648	GET_CPUINFO(%r1)
649	std	%r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)	/* free r27-r31 */
650	std	%r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
651	std	%r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
652	std	%r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
653	std	%r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
654	mfdar	%r30
655	mfdsisr	%r31
656	std	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
657	std	%r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
658	mfsprg1	%r1			/* restore SP, in case of branch */
659	mflr	%r28			/* save LR */
660	mfcr	%r29			/* save CR */
661
662	ld	%r31,TRAP_GENTRAP(0)
663	addi	%r31,%r31,(s_trap - generictrap)
664	mtlr	%r31
665
666	/* Put our exception vector in SPRG3 */
667	li	%r31, EXC_ALI
668	mtsprg3	%r31
669
670	/* Test whether we already had PR set */
671	mfsrr1	%r31
672	mtcr	%r31
673	blrl				/* Branch to s_trap */
674CNAME(aliend):
675
676/*
677 * Similar to the above for DSI
678 * Has to handle standard pagetable spills
679 */
680	.globl	CNAME(dsitrap),CNAME(dsiend)
681	.p2align 3
682CNAME(dsitrap):
683	mtsprg1	%r1			/* save SP */
684	GET_CPUINFO(%r1)
685	std	%r27,(PC_DISISAVE+CPUSAVE_R27)(%r1)	/* free r27-r31 */
686	std	%r28,(PC_DISISAVE+CPUSAVE_R28)(%r1)
687	std	%r29,(PC_DISISAVE+CPUSAVE_R29)(%r1)
688	std	%r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
689	std	%r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
690	mfcr	%r29			/* save CR */
691	mfxer	%r30			/* save XER */
692	mtsprg2	%r30			/* in SPRG2 */
693	mfsrr1	%r31			/* test kernel mode */
694	mtcr	%r31
695	mflr	%r28			/* save LR (SP already saved) */
696	ld	%r1,TRAP_GENTRAP(0)
697	addi	%r1,%r1,(disitrap-generictrap)
698	mtlr	%r1
699	blrl				/* Branch to disitrap */
700CNAME(dsiend):
701
702/*
703 * Preamble code for DSI/ISI traps
704 */
705disitrap:
706	/* Write the trap vector to SPRG3 by computing LR & 0xff00 */
707	mflr	%r1
708	andi.	%r1,%r1,0xff00
709	mtsprg3	%r1
710
711	GET_CPUINFO(%r1)
712	ld	%r31,(PC_DISISAVE+CPUSAVE_R27)(%r1)
713	std	%r31,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)
714	ld	%r30,(PC_DISISAVE+CPUSAVE_R28)(%r1)
715	std	%r30,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
716	ld	%r31,(PC_DISISAVE+CPUSAVE_R29)(%r1)
717	std	%r31,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
718	ld	%r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
719	std	%r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
720	ld	%r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
721	std	%r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
722	mfdar	%r30
723	mfdsisr	%r31
724	std	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
725	std	%r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
726
727#ifdef KDB
728	/* Try to detect a kernel stack overflow */
729	mfsrr1	%r31
730	mtcr	%r31
731	bt	17,realtrap		/* branch is user mode */
732	mfsprg1	%r31			/* get old SP */
733	clrrdi	%r31,%r31,12		/* Round SP down to nearest page */
734	sub.	%r30,%r31,%r30		/* SP - DAR */
735	bge	1f
736	neg	%r30,%r30		/* modulo value */
7371:	cmpldi	%cr0,%r30,4096		/* is DAR within a page of SP? */
738	bge	%cr0,realtrap		/* no, too far away. */
739
740	/* Now convert this DSI into a DDB trap.  */
741	GET_CPUINFO(%r1)
742	ld	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1) /* get DAR */
743	std	%r30,(PC_DBSAVE  +CPUSAVE_AIM_DAR)(%r1) /* save DAR */
744	ld	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1) /* get DSISR */
745	std	%r30,(PC_DBSAVE  +CPUSAVE_AIM_DSISR)(%r1) /* save DSISR */
746	ld	%r31,(PC_DISISAVE+CPUSAVE_R27)(%r1) /* get  r27 */
747	std	%r31,(PC_DBSAVE  +CPUSAVE_R27)(%r1) /* save r27 */
748	ld	%r30,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* get  r28 */
749	std	%r30,(PC_DBSAVE  +CPUSAVE_R28)(%r1) /* save r28 */
750	ld	%r31,(PC_DISISAVE+CPUSAVE_R29)(%r1) /* get  r29 */
751	std	%r31,(PC_DBSAVE  +CPUSAVE_R29)(%r1) /* save r29 */
752	ld	%r30,(PC_DISISAVE+CPUSAVE_R30)(%r1) /* get  r30 */
753	std	%r30,(PC_DBSAVE  +CPUSAVE_R30)(%r1) /* save r30 */
754	ld	%r31,(PC_DISISAVE+CPUSAVE_R31)(%r1) /* get  r31 */
755	std	%r31,(PC_DBSAVE  +CPUSAVE_R31)(%r1) /* save r31 */
756	b	dbtrap
757#endif
758
759	/* XXX need stack probe here */
760realtrap:
761/* Test whether we already had PR set */
762	mfsrr1	%r1
763	mtcr	%r1
764	mfsprg1	%r1			/* restore SP (might have been
765					   overwritten) */
766	bf	17,k_trap		/* branch if PSL_PR is false */
767	GET_CPUINFO(%r1)
768	ld	%r1,PC_CURPCB(%r1)
769	mr	%r27,%r28		/* Save LR, r29 */
770	mtsprg2	%r29
771	bl	restore_kernsrs		/* enable kernel mapping */
772	mfsprg2	%r29
773	mr	%r28,%r27
774	b	s_trap
775
776/*
777 * generictrap does some standard setup for trap handling to minimize
778 * the code that need be installed in the actual vectors. It expects
779 * the following conditions.
780 *
781 * R1 - Trap vector = LR & (0xff00 | R1)
782 * SPRG1 - Original R1 contents
783 * SPRG2 - Original LR
784 */
785
786generichypertrap:
787	mtsprg3 %r1
788	mfspr	%r1, SPR_HSRR0
789	mtsrr0	%r1
790	mfspr	%r1, SPR_HSRR1
791	mtsrr1	%r1
792	mfsprg3	%r1
793	.globl	CNAME(generictrap)
794generictrap:
795	/* Save R1 for computing the exception vector */
796	mtsprg3 %r1
797
798	/* Save interesting registers */
799	GET_CPUINFO(%r1)
800	std	%r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)	/* free r27-r31 */
801	std	%r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
802	std	%r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
803	std	%r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
804	std	%r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
805	mfdar	%r30
806	std	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
807	mfdsisr	%r30
808	std	%r30,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
809	mfsprg1	%r1			/* restore SP, in case of branch */
810	mfsprg2	%r28			/* save LR */
811	mfcr	%r29			/* save CR */
812
813	/* Compute the exception vector from the link register */
814	mfsprg3 %r31
815	ori	%r31,%r31,0xff00
816	mflr	%r30
817	addi	%r30,%r30,-4 /* The branch instruction, not the next */
818	and	%r30,%r30,%r31
819	mtsprg3	%r30
820
821	/* Test whether we already had PR set */
822	mfsrr1	%r31
823	mtcr	%r31
824
825s_trap:
826	bf	17,k_trap		/* branch if PSL_PR is false */
827	GET_CPUINFO(%r1)
828u_trap:
829	ld	%r1,PC_CURPCB(%r1)
830	mr	%r27,%r28		/* Save LR, r29 */
831	mtsprg2	%r29
832	bl	restore_kernsrs		/* enable kernel mapping */
833	mfsprg2	%r29
834	mr	%r28,%r27
835
836/*
837 * Now the common trap catching code.
838 */
839k_trap:
840	FRAME_SETUP(PC_TEMPSAVE)
841/* Call C interrupt dispatcher: */
842trapagain:
843	GET_TOCBASE(%r2)
844	addi	%r3,%r1,48
845	bl	CNAME(powerpc_interrupt)
846	nop
847
848	.globl	CNAME(trapexit)	/* backtrace code sentinel */
849CNAME(trapexit):
850/* Disable interrupts: */
851	mfmsr	%r3
852	andi.	%r3,%r3,~PSL_EE@l
853	mtmsr	%r3
854	isync
855/* Test AST pending: */
856	ld	%r5,FRAME_SRR1+48(%r1)
857	mtcr	%r5
858	bf	17,1f			/* branch if PSL_PR is false */
859
860	GET_CPUINFO(%r3)		/* get per-CPU pointer */
861	lwz	%r4, TD_FLAGS(%r13)	/* get thread flags value */
862	lis	%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
863	ori	%r5,%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
864	and.	%r4,%r4,%r5
865	beq	1f
866	mfmsr	%r3			/* re-enable interrupts */
867	ori	%r3,%r3,PSL_EE@l
868	mtmsr	%r3
869	isync
870	GET_TOCBASE(%r2)
871	addi	%r3,%r1,48
872	bl	CNAME(ast)
873	nop
874	.globl	CNAME(asttrapexit)	/* backtrace code sentinel #2 */
875CNAME(asttrapexit):
876	b	trapexit		/* test ast ret value ? */
8771:
878	FRAME_LEAVE(PC_TEMPSAVE)
879	rfid
880
881#if defined(KDB)
882/*
883 * Deliberate entry to dbtrap
884 */
885ASENTRY_NOPROF(breakpoint)
886	mtsprg1	%r1
887	mfmsr	%r3
888	mtsrr1	%r3
889	andi.	%r3,%r3,~(PSL_EE|PSL_ME)@l
890	mtmsr	%r3			/* disable interrupts */
891	isync
892	GET_CPUINFO(%r3)
893	std	%r27,(PC_DBSAVE+CPUSAVE_R27)(%r3)
894	std	%r28,(PC_DBSAVE+CPUSAVE_R28)(%r3)
895	std	%r29,(PC_DBSAVE+CPUSAVE_R29)(%r3)
896	std	%r30,(PC_DBSAVE+CPUSAVE_R30)(%r3)
897	std	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r3)
898	mflr	%r28
899	li	%r29,EXC_BPT
900	mtlr	%r29
901	mfcr	%r29
902	mtsrr0	%r28
903
904/*
905 * Now the kdb trap catching code.
906 */
907dbtrap:
908	/* Write the trap vector to SPRG3 by computing LR & 0xff00 */
909	mflr	%r1
910	andi.	%r1,%r1,0xff00
911	mtsprg3	%r1
912
913	GET_TOCBASE(%r1)			/* get new SP */
914	addis	%r1,%r1,TOC_REF(trapstk)@ha
915	ld	%r1,TOC_REF(trapstk)@l(%r1)
916	addi	%r1,%r1,(TRAPSTKSZ-48)
917
918	FRAME_SETUP(PC_DBSAVE)
919/* Call C trap code: */
920	GET_TOCBASE(%r2)
921	addi	%r3,%r1,48
922	bl	CNAME(db_trap_glue)
923	nop
924	or.	%r3,%r3,%r3
925	bne	dbleave
926/* This wasn't for KDB, so switch to real trap: */
927	ld	%r3,FRAME_EXC+48(%r1)	/* save exception */
928	GET_CPUINFO(%r4)
929	std	%r3,(PC_DBSAVE+CPUSAVE_R31)(%r4)
930	FRAME_LEAVE(PC_DBSAVE)
931	mtsprg1	%r1			/* prepare for entrance to realtrap */
932	GET_CPUINFO(%r1)
933	std	%r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)
934	std	%r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
935	std	%r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
936	std	%r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
937	std	%r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
938	mflr	%r28
939	mfcr	%r29
940	ld	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)
941	mtsprg3	%r31			/* SPRG3 was clobbered by FRAME_LEAVE */
942	mfsprg1	%r1
943	b	realtrap
944dbleave:
945	FRAME_LEAVE(PC_DBSAVE)
946	rfid
947
948/*
949 * In case of KDB we want a separate trap catcher for it
950 */
951	.globl	CNAME(dblow),CNAME(dbend)
952	.p2align 3
953CNAME(dblow):
954	mtsprg1	%r1			/* save SP */
955	mtsprg2	%r29			/* save r29 */
956	mfcr	%r29			/* save CR in r29 */
957	mfsrr1	%r1
958	mtcr	%r1
959	bf	17,1f			/* branch if privileged */
960
961	/* Unprivileged case */
962	mtcr	%r29			/* put the condition register back */
963        mfsprg2	%r29			/* ... and r29 */
964        mflr	%r1			/* save LR */
965	mtsprg2 %r1			/* And then in SPRG2 */
966
967	ld	%r1, TRAP_ENTRY(0)	/* Get branch address */
968	mtlr	%r1
969	li	%r1, 0	 		/* How to get the vector from LR */
970	blrl				/* Branch to generictrap */
971	/* No fallthrough */
9721:
973	GET_CPUINFO(%r1)
974	std	%r27,(PC_DBSAVE+CPUSAVE_R27)(%r1)	/* free r27 */
975	std	%r28,(PC_DBSAVE+CPUSAVE_R28)(%r1)	/* free r28 */
976        mfsprg2	%r28				/* r29 holds cr...  */
977        std	%r28,(PC_DBSAVE+CPUSAVE_R29)(%r1)	/* free r29 */
978        std	%r30,(PC_DBSAVE+CPUSAVE_R30)(%r1)	/* free r30 */
979        std	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)	/* free r31 */
980        mflr	%r28					/* save LR */
981	ld	%r1,TRAP_GENTRAP(0)
982	addi	%r1,%r1,(dbtrap-generictrap)
983	mtlr	%r1
984	blrl				/* Branch to dbtrap */
985CNAME(dbend):
986#endif /* KDB */
987