xref: /freebsd/sys/powerpc/booke/trap_subr.S (revision 2f6a179eb910129fb812c1ad1bdc300da1203dc0)
1/*-
2 * Copyright (C) 2006 Semihalf, Marian Balakowicz <m8@semihalf.com>
3 * Copyright (C) 2006 Semihalf, Rafal Jaworowski <raj@semihalf.com>
4 * Copyright (C) 2006 Juniper Networks, Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 *    derived from this software without specific prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
21 * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
23 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 *
29 * $FreeBSD$
30 */
31/*-
32 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
33 * Copyright (C) 1995, 1996 TooLs GmbH.
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 *    notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 *    notice, this list of conditions and the following disclaimer in the
43 *    documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 *    must display the following acknowledgement:
46 *	This product includes software developed by TooLs GmbH.
47 * 4. The name of TooLs GmbH may not be used to endorse or promote products
48 *    derived from this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
51 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
52 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
53 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
54 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
55 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
56 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
57 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
58 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
59 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
60 *
61 *	from: $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $
62 */
63
64/*
65 * NOTICE: This is not a standalone file.  to use it, #include it in
66 * your port's locore.S, like so:
67 *
68 *	#include <powerpc/booke/trap_subr.S>
69 */
70
71/*
72 * SPRG usage notes
73 *
74 * SPRG0 - pcpu pointer
75 * SPRG1 - all interrupts except TLB miss, critical, machine check
76 * SPRG2 - critical
77 * SPRG3 - machine check
78 *
79 */
80
81/* Get the per-CPU data structure */
82#define GET_CPUINFO(r) mfsprg0 r
83
84/*
85 * Standard interrupt prolog
86 *
87 * sprg_sp - SPRG{1-3} reg used to temporarily store the SP
88 * savearea - temp save area (pc_{tempsave, disisave, critsave, mchksave})
89 * isrr0-1 - save restore registers with CPU state at interrupt time (may be
90 *           SRR0-1, CSRR0-1, MCSRR0-1
91 *
92 * 1. saves in the given savearea:
93 *   - R30-31
94 *   - DEAR, ESR
95 *   - xSRR0-1
96 *
97 * 2. saves CR -> R30
98 *
99 * 3. switches to kstack if needed
100 *
101 * 4. notes:
102 *   - R31 can be used as scratch register until a new frame is layed on
103 *     the stack with FRAME_SETUP
104 *
105 *   - potential TLB miss: NO. Saveareas are always acessible via TLB1
106 *     permanent entries, and within this prolog we do not dereference any
107 *     locations potentially not in the TLB
108 */
109#define STANDARD_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
110	mtspr	sprg_sp, %r1;		/* Save SP */			\
111	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
112	stw	%r30, (savearea+CPUSAVE_R30)(%r1);			\
113	stw	%r31, (savearea+CPUSAVE_R31)(%r1); 			\
114	mfdear	%r30;		 					\
115	mfesr	%r31;							\
116	stw	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1); 		\
117	stw	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1); 		\
118	mfspr	%r30, isrr0;						\
119	mfspr	%r31, isrr1;	 	/* MSR at interrupt time */	\
120	stw	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
121	stw	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
122	isync;			 					\
123	mfspr	%r1, sprg_sp;	 	/* Restore SP */		\
124	mfcr	%r30;		 	/* Save CR */			\
125	/* switch to per-thread kstack if intr taken in user mode */	\
126	mtcr	%r31;			/* MSR at interrupt time  */	\
127	bf	17, 1f;							\
128	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
129	lwz	%r1, PC_CURPCB(%r1); 	/* Per-thread kernel stack */	\
1301:
131
132#define	STANDARD_CRIT_PROLOG(sprg_sp, savearea, isrr0, isrr1)		\
133	mtspr	sprg_sp, %r1;		/* Save SP */			\
134	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
135	stw	%r30, (savearea+CPUSAVE_R30)(%r1);			\
136	stw	%r31, (savearea+CPUSAVE_R31)(%r1);			\
137	mfdear	%r30;							\
138	mfesr	%r31;							\
139	stw	%r30, (savearea+CPUSAVE_BOOKE_DEAR)(%r1);		\
140	stw	%r31, (savearea+CPUSAVE_BOOKE_ESR)(%r1);		\
141	mfspr	%r30, isrr0;						\
142	mfspr	%r31, isrr1;		/* MSR at interrupt time */	\
143	stw	%r30, (savearea+CPUSAVE_SRR0)(%r1);			\
144	stw	%r31, (savearea+CPUSAVE_SRR1)(%r1);			\
145	mfspr	%r30, SPR_SRR0;						\
146	mfspr	%r31, SPR_SRR1;		/* MSR at interrupt time */	\
147	stw	%r30, (savearea+CPUSAVE_SRR0+8)(%r1);			\
148	stw	%r31, (savearea+CPUSAVE_SRR1+8)(%r1);			\
149	isync;								\
150	mfspr	%r1, sprg_sp;		/* Restore SP */		\
151	mfcr	%r30;			/* Save CR */			\
152	/* switch to per-thread kstack if intr taken in user mode */	\
153	mtcr	%r31;			/* MSR at interrupt time  */	\
154	bf	17, 1f;							\
155	GET_CPUINFO(%r1);		/* Per-cpu structure */		\
156	lwz	%r1, PC_CURPCB(%r1);	/* Per-thread kernel stack */	\
1571:
158
159/*
160 * FRAME_SETUP assumes:
161 *	SPRG{1-3}	SP at the time interrupt occured
162 *	savearea	r30-r31, DEAR, ESR, xSRR0-1
163 *	r30		CR
164 *	r31		scratch
165 *	r1		kernel stack
166 *
167 * sprg_sp - SPRG reg containing SP at the time interrupt occured
168 * savearea - temp save
169 * exc - exception number (EXC_xxx)
170 *
171 * 1. sets a new frame
172 * 2. saves in the frame:
173 *   - R0, R1 (SP at the time of interrupt), R2, LR, CR
174 *   - R3-31 (R30-31 first restored from savearea)
175 *   - XER, CTR, DEAR, ESR (from savearea), xSRR0-1
176 *
177 * Notes:
178 * - potential TLB miss: YES, since we make dereferences to kstack, which
179 *   can happen not covered (we can have up to two DTLB misses if fortunate
180 *   enough i.e. when kstack crosses page boundary and both pages are
181 *   untranslated)
182 */
183#define	FRAME_SETUP(sprg_sp, savearea, exc)				\
184	mfspr	%r31, sprg_sp;		/* get saved SP */		\
185	/* establish a new stack frame and put everything on it */	\
186	stwu	%r31, -FRAMELEN(%r1);					\
187	stw	%r0, FRAME_0+8(%r1);	/* save r0 in the trapframe */	\
188	stw	%r31, FRAME_1+8(%r1);	/* save SP   "      "       */	\
189	stw	%r2, FRAME_2+8(%r1);	/* save r2   "      "       */	\
190	mflr	%r31;		 					\
191	stw	%r31, FRAME_LR+8(%r1);	/* save LR   "      "       */	\
192	stw	%r30, FRAME_CR+8(%r1);	/* save CR   "      "       */	\
193	GET_CPUINFO(%r2);						\
194	lwz	%r30, (savearea+CPUSAVE_R30)(%r2); /* get saved r30 */	\
195	lwz	%r31, (savearea+CPUSAVE_R31)(%r2); /* get saved r31 */	\
196	/* save R3-31 */						\
197	stmw	%r3,  FRAME_3+8(%r1) ;					\
198	/* save DEAR, ESR */						\
199	lwz	%r28, (savearea+CPUSAVE_BOOKE_DEAR)(%r2);		\
200	lwz	%r29, (savearea+CPUSAVE_BOOKE_ESR)(%r2);		\
201	stw	%r28, FRAME_BOOKE_DEAR+8(1);				\
202	stw	%r29, FRAME_BOOKE_ESR+8(1);				\
203	/* save XER, CTR, exc number */					\
204	mfxer	%r3;							\
205	mfctr	%r4;							\
206	stw	%r3, FRAME_XER+8(1);					\
207	stw	%r4, FRAME_CTR+8(1);					\
208	li	%r5, exc;						\
209	stw	%r5, FRAME_EXC+8(1);					\
210	/* save xSSR0-1 */						\
211	lwz	%r30, (savearea+CPUSAVE_SRR0)(%r2);			\
212	lwz	%r31, (savearea+CPUSAVE_SRR1)(%r2);			\
213	stw	%r30, FRAME_SRR0+8(1);					\
214	stw	%r31, FRAME_SRR1+8(1)
215
216
217/*
218 *
219 * isrr0-1 - save restore registers to restore CPU state to (may be
220 *           SRR0-1, CSRR0-1, MCSRR0-1
221 *
222 * Notes:
223 *  - potential TLB miss: YES. The deref'd kstack may be not covered
224 */
225#define	FRAME_LEAVE(isrr0, isrr1)					\
226	/* restore CTR, XER, LR, CR */					\
227	lwz	%r4, FRAME_CTR+8(%r1);					\
228	lwz	%r5, FRAME_XER+8(%r1);					\
229	lwz	%r6, FRAME_LR+8(%r1);					\
230	lwz	%r7, FRAME_CR+8(%r1);					\
231	mtctr	%r4;							\
232	mtxer	%r5;							\
233	mtlr	%r6;							\
234	mtcr	%r7;							\
235	/* restore xSRR0-1 */						\
236	lwz	%r30, FRAME_SRR0+8(%r1);				\
237	lwz	%r31, FRAME_SRR1+8(%r1);				\
238	mtspr	isrr0, %r30;						\
239	mtspr	isrr1, %r31;						\
240	/* restore R2-31, SP */						\
241	lmw	%r2, FRAME_2+8(%r1) ;					\
242	lwz	%r0, FRAME_0+8(%r1);					\
243	lwz	%r1, FRAME_1+8(%r1);					\
244	isync
245
246/*
247 * TLB miss prolog
248 *
249 * saves LR, CR, SRR0-1, R20-31 in the TLBSAVE area
250 *
251 * Notes:
252 *  - potential TLB miss: NO. It is crucial that we do not generate a TLB
253 *    miss withing the TLB prolog itself!
254 *  - TLBSAVE is always translated
255 */
256#define TLB_PROLOG							\
257	mtsprg4	%r1;			/* Save SP */			\
258	mtsprg5 %r28;							\
259	mtsprg6 %r29;							\
260	/* calculate TLB nesting level and TLBSAVE instance address */	\
261	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
262	lwz	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
263	rlwinm	%r29, %r28, 6, 24, 25;	/* 4 x TLBSAVE_LEN */		\
264	addi	%r28, %r28, 1;						\
265	stw	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
266	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l; 			\
267	add	%r1, %r1, %r29;		/* current TLBSAVE ptr */	\
268									\
269	/* save R20-31 */						\
270	mfsprg5 %r28;		 					\
271	mfsprg6 %r29;							\
272	stmw	%r20, (TLBSAVE_BOOKE_R20)(%r1);				\
273	/* save LR, CR */						\
274	mflr	%r30;		 					\
275	mfcr	%r31;							\
276	stw	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
277	stw	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
278	/* save SRR0-1 */						\
279	mfsrr0	%r30;		/* execution addr at interrupt time */	\
280	mfsrr1	%r31;		/* MSR at interrupt time*/		\
281	stw	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);	/* save SRR0 */	\
282	stw	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);	/* save SRR1 */	\
283	isync;								\
284	mfsprg4	%r1
285
286/*
287 * restores LR, CR, SRR0-1, R20-31 from the TLBSAVE area
288 *
289 * same notes as for the TLB_PROLOG
290 */
291#define TLB_RESTORE							\
292	mtsprg4	%r1;			/* Save SP */			\
293	GET_CPUINFO(%r1);	 	/* Per-cpu structure */		\
294	/* calculate TLB nesting level and TLBSAVE instance addr */	\
295	lwz	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
296	subi	%r28, %r28, 1;						\
297	stw	%r28, PC_BOOKE_TLB_LEVEL(%r1);				\
298	rlwinm	%r29, %r28, 6, 24, 25; /* 4 x TLBSAVE_LEN */		\
299	addi	%r29, %r29, PC_BOOKE_TLBSAVE@l;				\
300	add	%r1, %r1, %r29;						\
301									\
302	/* restore LR, CR */						\
303	lwz	%r30, (TLBSAVE_BOOKE_LR)(%r1);				\
304	lwz	%r31, (TLBSAVE_BOOKE_CR)(%r1);				\
305	mtlr	%r30;							\
306	mtcr	%r31;							\
307	/* restore SRR0-1 */						\
308	lwz	%r30, (TLBSAVE_BOOKE_SRR0)(%r1);			\
309	lwz	%r31, (TLBSAVE_BOOKE_SRR1)(%r1);			\
310	mtsrr0	%r30;							\
311	mtsrr1	%r31;							\
312	/* restore R20-31 */						\
313	lmw	%r20, (TLBSAVE_BOOKE_R20)(%r1);				\
314	mfsprg4	%r1
315
316
317#define INTERRUPT(label)						\
318	.globl	label;							\
319	.align	5;							\
320	CNAME(label):
321
322/*
323 * Interrupt handling routines in BookE can be flexibly placed and do not have
324 * to live in pre-defined vectors location. Note they need to be TLB-mapped at
325 * all times in order to be able to handle exceptions. We thus arrange for
326 * them to be part of kernel text which is always TLB-accessible.
327 *
328 * The interrupt handling routines have to be 16 bytes aligned: we align them
329 * to 32 bytes (cache line length) which supposedly performs better.
330 *
331 */
332	.text
333	.globl CNAME(interrupt_vector_base)
334	.align 5
335interrupt_vector_base:
336
337/****************************************
338 * Critical input interrupt
339 ****************************************/
340INTERRUPT(int_critical_input)
341	STANDARD_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
342	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_CRIT)
343	addi	%r3, %r1, 8
344	bl	CNAME(powerpc_crit_interrupt)
345	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
346	rfci
347
348
349/****************************************
350 * Machine check interrupt
351 ****************************************/
352INTERRUPT(int_machine_check)
353	STANDARD_PROLOG(SPR_SPRG3, PC_BOOKE_MCHKSAVE, SPR_MCSRR0, SPR_MCSRR1)
354	FRAME_SETUP(SPR_SPRG3, PC_BOOKE_MCHKSAVE, EXC_MCHK)
355	addi	%r3, %r1, 8
356	bl	CNAME(powerpc_mchk_interrupt)
357	FRAME_LEAVE(SPR_MCSRR0, SPR_MCSRR1)
358	rfmci
359
360
361/****************************************
362 * Data storage interrupt
363 ****************************************/
364INTERRUPT(int_data_storage)
365	STANDARD_PROLOG(SPR_SPRG1, PC_DISISAVE, SPR_SRR0, SPR_SRR1)
366	FRAME_SETUP(SPR_SPRG1, PC_DISISAVE, EXC_DSI)
367	b	trap_common
368
369
370/****************************************
371 * Instruction storage interrupt
372 ****************************************/
373INTERRUPT(int_instr_storage)
374	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
375	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ISI)
376	b	trap_common
377
378
379/****************************************
380 * External input interrupt
381 ****************************************/
382INTERRUPT(int_external_input)
383	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
384	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_EXI)
385	bl	CNAME(powerpc_extr_interrupt)
386	b	trapexit
387
388
389INTERRUPT(int_alignment)
390	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
391	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_ALI)
392	b	trap_common
393
394
395INTERRUPT(int_program)
396	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
397	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_PGM)
398	b	trap_common
399
400
401/****************************************
402 * System call
403 ****************************************/
404INTERRUPT(int_syscall)
405	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
406	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_SC)
407	b	trap_common
408
409
410/****************************************
411 * Decrementer interrupt
412 ****************************************/
413INTERRUPT(int_decrementer)
414	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
415	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_DECR)
416	addi	%r3, %r1, 8
417	bl	CNAME(powerpc_decr_interrupt)
418	b	trapexit
419
420
421/****************************************
422 * Fixed interval timer
423 ****************************************/
424INTERRUPT(int_fixed_interval_timer)
425	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
426	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_FIT)
427	b	trap_common
428
429
430/****************************************
431 * Watchdog interrupt
432 ****************************************/
433INTERRUPT(int_watchdog)
434	STANDARD_PROLOG(SPR_SPRG1, PC_TEMPSAVE, SPR_SRR0, SPR_SRR1)
435	FRAME_SETUP(SPR_SPRG1, PC_TEMPSAVE, EXC_WDOG)
436	b	trap_common
437
438
439/****************************************
440 * Data TLB miss interrupt
441 *
442 * There can be nested TLB misses - while
443 * handling a TLB miss we dereference data
444 * structures that may be not covered by
445 * translations. We support up to
446 * TLB_NESTED_MAX-1 nested misses.
447 *
448 * Registers use:
449 *	r31 - dear
450 *	r30 - tlb0 entry address
451 *	r29 - saved mas0
452 *	r28 - saved mas1
453 *	r27 - saved mas2
454 *	r26 - pmap address
455 *	r25 - pte address
456 *
457 *	r20:r23 - scratch registers
458 ****************************************/
459INTERRUPT(int_data_tlb_error)
460	TLB_PROLOG
461
462	mfdear	%r31
463
464	/*
465	 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
466	 * lookup overwriting current contents (which was hw filled).
467	 */
468	mfspr	%r29, SPR_MAS0
469	mfspr	%r28, SPR_MAS1
470	mfspr	%r27, SPR_MAS2
471
472	/* return tlb0 entry address in r30 */
473	bl	get_tlb0table_entry
474
475	/* Check faulting address. */
476	lis	%r21, VM_MAXUSER_ADDRESS@h
477	ori	%r21, %r21, VM_MAXUSER_ADDRESS@l
478	cmplw	cr0, %r31, %r21
479	blt	search_user_pmap
480
481	/* If it's kernel address, allow only supervisor mode misses. */
482	mfsrr1	%r21
483	mtcr	%r21
484	bt	17, search_failed	/* check MSR[PR] */
485
486search_kernel_pmap:
487	/* Load r26 with kernel_pmap address */
488	lis	%r26, kernel_pmap_store@h
489	ori	%r26, %r26, kernel_pmap_store@l
490
491	/* Force kernel tid, set TID to 0 in MAS1. */
492	li	%r21, 0
493	rlwimi	%r28, %r21, 0, 8, 15	/* clear TID bits */
494
495tlb_miss_handle:
496	/* This may result in nested tlb miss. */
497	bl	pte_lookup		/* returns PTE address in R25 */
498
499	cmpwi	%r25, 0			/* pte found? */
500	beq	search_failed
501
502	/* Finish up, write TLB entry. */
503	bl	tlb_fill_entry
504
505tlb_miss_return:
506	TLB_RESTORE
507	rfi
508
509search_user_pmap:
510	/* Load r26 with current user space process pmap */
511	GET_CPUINFO(%r26)
512	lwz	%r26, PC_CURPMAP(%r26)
513
514	b	tlb_miss_handle
515
516search_failed:
517	/*
518	 * Whenever we don't find a TLB mapping in PT, set a TLB0 entry with
519	 * the faulting virtual address anyway, but put a fake RPN and no
520	 * access rights. This should cause a following {D,I}SI exception.
521	 */
522	lis	%r23, 0xffff0000@h	/* revoke all permissions */
523
524	/* Save MAS registers to tlb0[] table. */
525	stw	%r28, TLB0TABLE_MAS1(%r30)	/* write tlb0[idx].mas1 */
526	stw	%r27, TLB0TABLE_MAS2(%r30)	/* write tlb0[idx].mas2 */
527	stw	%r23, TLB0TABLE_MAS3(%r30)	/* write tlb0[idx].mas3 */
528
529	/* Load MAS registers. */
530	mtspr	SPR_MAS0, %r29
531	isync
532	mtspr	SPR_MAS1, %r28
533	isync
534	mtspr	SPR_MAS2, %r27
535	isync
536	mtspr	SPR_MAS3, %r23
537	isync
538
539	tlbwe
540	msync
541	isync
542	b	tlb_miss_return
543
544/******************************************************/
545/*
546 * Calculate address of tlb0[tlb0table_idx], save it in r30
547 *
548 * tlb0table_idx = (way * entries_per_way) + entry_number
549 *		entries_per_way = 128
550 *		entry_number is defined by EPN[45:51]
551 *
552 * input: r31 - faulting address
553 * input: r29 - MAS0
554 * output: r30 - address of corresponding tlb0[] entry
555 *
556 * scratch regs used: r21-r23
557 */
558/******************************************************/
559get_tlb0table_entry:
560	lis	%r21, 0				/* keeps tlb0table_idx */
561
562	/* Add entry number, use DEAR from r31 (faulting va) */
563	rlwinm	%r22, %r31, 20, 25, 31		/* get EPN[45:51] */
564	add	%r21, %r21, %r22
565
566	/* Select way */
567	rlwinm	%r22, %r29, 16, 30, 31		/* get way# = ESEL[0:1] */
568
569	/* Get number of entries per tlb0 way. */
570	lis	%r23, tlb0_nentries_per_way@h
571	ori	%r23, %r23, tlb0_nentries_per_way@l
572	lwz	%r23, 0(%r23)
573
574	mullw	%r22, %r22, %r23		/* multiply by #entries per way */
575	add	%r21, %r21, %r22
576
577	mulli	%r21, %r21, TLB0_ENTRY_SIZE	/* multipy by tlb0 entry size */
578
579	/* Get tlb0[tlb0tble_idx] address, save it in r30 */
580	lis	%r30, tlb0@h
581	ori	%r30, %r30, tlb0@l
582	lwz	%r30, 0(%r30)
583	add	%r30, %r30, %r21
584	blr
585
586
587/******************************************************/
588/*
589 * Return pte address that corresponds to given pmap/va.
590 * If there is no valid entry return 0.
591 *
592 * input: r26 - pmap
593 * input: r31 - dear
594 * output: r25 - pte address
595 *
596 * scratch regs used: r21
597 */
598/******************************************************/
599pte_lookup:
600	cmpwi	%r26, 0
601	beq	1f			/* fail quickly if pmap is invalid */
602
603	srwi	%r21, %r31, PDIR_SHIFT		/* pdir offset */
604	slwi	%r21, %r21, PDIR_ENTRY_SHIFT	/* multiply by pdir entry size */
605
606	addi	%r25, %r26, PM_PDIR	/* pmap pm_dir[] address */
607	add	%r25, %r25, %r21	/* offset within pm_pdir[] table */
608	lwz	%r25, 0(%r25)		/* get ptbl address, i.e. pmap->pm_pdir[pdir_idx] */
609
610	cmpwi	%r25, 0
611	beq	2f
612
613	lis	%r21, PTBL_MASK@h
614	ori	%r21, %r21, PTBL_MASK@l
615	and	%r21, %r21, %r31
616
617	/* ptbl offset, multiply by ptbl entry size */
618	srwi	%r21, %r21, (PTBL_SHIFT - PTBL_ENTRY_SHIFT)
619
620	add	%r25, %r25, %r21		/* address of pte entry */
621	lwz	%r21, PTE_FLAGS(%r25)		/* get pte->flags */
622	andis.	%r21, %r21, PTE_VALID@h
623	bne	2f
6241:
625	li	%r25, 0
6262:
627	blr
628
629/******************************************************/
630/*
631 * Save MAS1-MAS3 registers to tlb0[] table, write TLB entry
632 *
633 * input:
634 * r29 - mas0
635 * r28 - mas1
636 * r27 - mas2
637 * r25 - pte
638 * r30 - tlb0 entry address
639 *
640 * output: none
641 *
642 * scratch regs: r21-r23
643 */
644/******************************************************/
645tlb_fill_entry:
646	/* Handle pte flags. */
647	lwz	%r21, PTE_FLAGS(%r25)		/* get pte->flags */
648	oris	%r21, %r21, PTE_REFERENCED@h	/* set referenced bit */
649
650	andi.	%r22, %r21, (PTE_UW | PTE_UW)@l	/* check if writable */
651	beq	1f
652	oris	%r21, %r21, PTE_MODIFIED@h	/* set modified bit */
6531:
654	stw	%r21, PTE_FLAGS(%r25)		/* write it back */
655
656	/* Update MAS2. */
657	rlwimi	%r27, %r21, 0, 27, 30		/* insert WIMG bits from pte */
658
659	/* Setup MAS3 value in r23. */
660	lwz	%r23, PTE_RPN(%r25)		/* get pte->rpn */
661
662	rlwimi	%r23, %r21, 24, 26, 31		/* insert protection bits from pte */
663
664	/* Save MAS registers to tlb0[] table. */
665	stw	%r28, TLB0TABLE_MAS1(%r30)	/* write tlb0[idx].mas1 */
666	stw	%r27, TLB0TABLE_MAS2(%r30)	/* write tlb0[idx].mas2 */
667	stw	%r23, TLB0TABLE_MAS3(%r30)	/* write tlb0[idx].mas3 */
668
669	/* Load MAS registers. */
670	mtspr	SPR_MAS0, %r29
671	isync
672	mtspr	SPR_MAS1, %r28
673	isync
674	mtspr	SPR_MAS2, %r27
675	isync
676	mtspr	SPR_MAS3, %r23
677	isync
678
679	tlbwe
680	isync
681	msync
682	blr
683
684/****************************************
685 * Instruction TLB miss interrupt
686 *
687 * Same notes as for the Data TLB miss
688 *
689 ****************************************/
690INTERRUPT(int_inst_tlb_error)
691	TLB_PROLOG
692
693	mfsrr0	%r31			/* faulting address */
694
695	/*
696	 * Save MAS0-MAS2 registers. There might be another tlb miss during pte
697	 * lookup overwriting current contents (which was hw filled).
698	 */
699	mfspr	%r29, SPR_MAS0
700	mfspr	%r28, SPR_MAS1
701	mfspr	%r27, SPR_MAS2
702
703	/* return tlb0 entry address in r30 */
704	bl	get_tlb0table_entry
705
706	mfsrr1	%r21
707	mtcr	%r21
708
709	/* check MSR[PR] */
710	bt	17, search_user_pmap
711	b	search_kernel_pmap
712
713
714	.globl	interrupt_vector_top
715interrupt_vector_top:
716
717/****************************************
718 * Debug interrupt
719 ****************************************/
720INTERRUPT(int_debug)
721	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
722	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
723	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r2);
724	lis	%r4, interrupt_vector_base@ha
725	addi	%r4, %r4, interrupt_vector_base@l
726	cmplw	cr0, %r3, %r4
727	blt	1f
728	lis	%r4, interrupt_vector_top@ha
729	addi	%r4, %r4, interrupt_vector_top@l
730	cmplw	cr0, %r3, %r4
731	bge	1f
732	/* Disable single-stepping for the interrupt handlers. */
733	lwz	%r3, FRAME_SRR1+8(%r1);
734	rlwinm	%r3,%r3,0,23,21
735	stw	%r3, FRAME_SRR1+8(%r1);
736	/* Restore srr0 and srr1 as they could have been clobbered. */
737	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r2);
738	mtspr	SPR_SRR0, %r3
739	lwz	%r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r2);
740	mtspr	SPR_SRR1, %r4
741	b	9f
7421:
743	addi	%r3, %r1, 8
744	bl	CNAME(trap)
745	/*
746	 * Handle ASTs, needed for proper support of single-stepping.
747	 * We actually need to return to the process with an rfi.
748	 */
749	b	trapexit
7509:
751	FRAME_LEAVE(SPR_CSRR0, SPR_CSRR1)
752	rfci
753
754
755/********************************
756 * Common trap code
757 ********************************/
758trap_common:
759	/* Call C trap dispatcher */
760	addi	%r3, %r1, 8
761	bl	CNAME(trap)
762
763	.globl	CNAME(trapexit)		/* exported for db_backtrace use */
764CNAME(trapexit):
765	/* disable interrupts */
766	wrteei	0
767
768	/* Test AST pending - makes sense for user process only */
769	lwz	%r5, FRAME_SRR1+8(%r1)
770	mtcr	%r5
771	bf	17, 1f
772
773	GET_CPUINFO(%r3)
774	lwz	%r4, PC_CURTHREAD(%r3)
775	lwz	%r4, TD_FLAGS(%r4)
776	lis	%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
777	ori	%r5, %r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
778	and.	%r4, %r4, %r5
779	beq	1f
780
781	/* re-enable interrupts before calling ast() */
782	wrteei	1
783
784	addi	%r3, %r1, 8
785	bl	CNAME(ast)
786	.globl	CNAME(asttrapexit)	/* db_backtrace code sentinel #2 */
787CNAME(asttrapexit):
788	b	trapexit		/* test ast ret value ? */
7891:
790	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
791	rfi
792
793
794#if defined(KDB)
795/*
796 * Deliberate entry to dbtrap
797 */
798	.globl	CNAME(breakpoint)
799CNAME(breakpoint):
800	mtsprg1	%r1
801	mfmsr	%r3
802	mtsrr1	%r3
803	andi.	%r3,%r3,~(PSL_EE|PSL_ME)@l
804	mtmsr	%r3			/* disable interrupts */
805	isync
806	GET_CPUINFO(%r3)
807	stw	%r30,(PC_DBSAVE+CPUSAVE_R30)(%r3)
808	stw	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r3)
809
810	mflr	%r31
811	mtsrr0	%r31
812
813	mfdear	%r30
814	mfesr	%r31
815	stw	%r30, (PC_DBSAVE+CPUSAVE_BOOKE_DEAR)(%r3)
816	stw	%r31, (PC_DBSAVE+CPUSAVE_BOOKE_ESR)(%r3)
817
818	mfsrr0	%r30
819	mfsrr1	%r31
820	stw	%r30, (PC_DBSAVE+CPUSAVE_SRR0)(%r3)
821	stw	%r31, (PC_DBSAVE+CPUSAVE_SRR1)(%r3)
822	isync
823
824	mfcr	%r30
825
826/*
827 * Now the kdb trap catching code.
828 */
829dbtrap:
830	FRAME_SETUP(SPR_SPRG1, PC_DBSAVE, EXC_DEBUG)
831/* Call C trap code: */
832	addi	%r3,%r1,8
833	bl	CNAME(db_trap_glue)
834	or.	%r3,%r3,%r3
835	bne	dbleave
836/* This wasn't for KDB, so switch to real trap: */
837	b	trap_common
838
839dbleave:
840	FRAME_LEAVE(SPR_SRR0, SPR_SRR1)
841	rfi
842#endif /* KDB */
843