xref: /linux/arch/powerpc/kernel/exceptions-64s.S (revision b889fcf63cb62e7fdb7816565e28f44dbe4a76a5)
1/*
2 * This file contains the 64-bit "server" PowerPC variant
3 * of the low level exception handling including exception
4 * vectors, exception return, part of the slb and stab
5 * handling and other fixed offset specific things.
6 *
7 * This file is meant to be #included from head_64.S due to
8 * position dependent assembly.
9 *
10 * Most of this originates from head_64.S and thus has the same
11 * copyright history.
12 *
13 */
14
15#include <asm/hw_irq.h>
16#include <asm/exception-64s.h>
17#include <asm/ptrace.h>
18
19/*
20 * We layout physical memory as follows:
21 * 0x0000 - 0x00ff : Secondary processor spin code
22 * 0x0100 - 0x17ff : pSeries Interrupt prologs
23 * 0x1800 - 0x4000 : interrupt support common interrupt prologs
24 * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
25 * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
26 * 0x7000 - 0x7fff : FWNMI data area
27 * 0x8000 - 0x8fff : Initial (CPU0) segment table
28 * 0x9000 -        : Early init and support code
29 */
30	/* Syscall routine is used twice, in reloc-off and reloc-on paths */
31#define SYSCALL_PSERIES_1 					\
32BEGIN_FTR_SECTION						\
33	cmpdi	r0,0x1ebe ; 					\
34	beq-	1f ;						\
35END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE)				\
36	mr	r9,r13 ;					\
37	GET_PACA(r13) ;						\
38	mfspr	r11,SPRN_SRR0 ;					\
390:
40
41#define SYSCALL_PSERIES_2_RFID 					\
42	mfspr	r12,SPRN_SRR1 ;					\
43	ld	r10,PACAKBASE(r13) ; 				\
44	LOAD_HANDLER(r10, system_call_entry) ; 			\
45	mtspr	SPRN_SRR0,r10 ; 				\
46	ld	r10,PACAKMSR(r13) ;				\
47	mtspr	SPRN_SRR1,r10 ; 				\
48	rfid ; 							\
49	b	. ;	/* prevent speculative execution */
50
51#define SYSCALL_PSERIES_3					\
52	/* Fast LE/BE switch system call */			\
531:	mfspr	r12,SPRN_SRR1 ;					\
54	xori	r12,r12,MSR_LE ;				\
55	mtspr	SPRN_SRR1,r12 ;					\
56	rfid ;		/* return to userspace */		\
57	b	. ;						\
582:	mfspr	r12,SPRN_SRR1 ;					\
59	andi.	r12,r12,MSR_PR ;				\
60	bne	0b ;						\
61	mtspr	SPRN_SRR0,r3 ;					\
62	mtspr	SPRN_SRR1,r4 ;					\
63	mtspr	SPRN_SDR1,r5 ;					\
64	rfid ;							\
65	b	. ;	/* prevent speculative execution */
66
67#if defined(CONFIG_RELOCATABLE)
68	/*
69	 * We can't branch directly; in the direct case we use LR
70	 * and system_call_entry restores LR.  (We thus need to move
71	 * LR to r10 in the RFID case too.)
72	 */
73#define SYSCALL_PSERIES_2_DIRECT				\
74	mflr	r10 ;						\
75	ld	r12,PACAKBASE(r13) ; 				\
76	LOAD_HANDLER(r12, system_call_entry_direct) ;		\
77	mtlr	r12 ;						\
78	mfspr	r12,SPRN_SRR1 ;					\
79	/* Re-use of r13... No spare regs to do this */	\
80	li	r13,MSR_RI ;					\
81	mtmsrd 	r13,1 ;						\
82	GET_PACA(r13) ;	/* get r13 back */			\
83	blr ;
84#else
85	/* We can branch directly */
86#define SYSCALL_PSERIES_2_DIRECT				\
87	mfspr	r12,SPRN_SRR1 ;					\
88	li	r10,MSR_RI ;					\
89	mtmsrd 	r10,1 ;			/* Set RI (EE=0) */	\
90	b	system_call_entry_direct ;
91#endif
92
93/*
94 * This is the start of the interrupt handlers for pSeries
95 * This code runs with relocation off.
96 * Code from here to __end_interrupts gets copied down to real
97 * address 0x100 when we are running a relocatable kernel.
98 * Therefore any relative branches in this section must only
99 * branch to labels in this section.
100 */
101	. = 0x100
102	.globl __start_interrupts
103__start_interrupts:
104
105	.globl system_reset_pSeries;
106system_reset_pSeries:
107	HMT_MEDIUM;
108	SET_SCRATCH0(r13)
109#ifdef CONFIG_PPC_P7_NAP
110BEGIN_FTR_SECTION
111	/* Running native on arch 2.06 or later, check if we are
112	 * waking up from nap. We only handle no state loss and
113	 * supervisor state loss. We do -not- handle hypervisor
114	 * state loss at this time.
115	 */
116	mfspr	r13,SPRN_SRR1
117	rlwinm.	r13,r13,47-31,30,31
118	beq	9f
119
120	/* waking up from powersave (nap) state */
121	cmpwi	cr1,r13,2
122	/* Total loss of HV state is fatal, we could try to use the
123	 * PIR to locate a PACA, then use an emergency stack etc...
124	 * but for now, let's just stay stuck here
125	 */
126	bgt	cr1,.
127	GET_PACA(r13)
128
129#ifdef CONFIG_KVM_BOOK3S_64_HV
130	li	r0,KVM_HWTHREAD_IN_KERNEL
131	stb	r0,HSTATE_HWTHREAD_STATE(r13)
132	/* Order setting hwthread_state vs. testing hwthread_req */
133	sync
134	lbz	r0,HSTATE_HWTHREAD_REQ(r13)
135	cmpwi	r0,0
136	beq	1f
137	b	kvm_start_guest
1381:
139#endif
140
141	beq	cr1,2f
142	b	.power7_wakeup_noloss
1432:	b	.power7_wakeup_loss
1449:
145END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
146#endif /* CONFIG_PPC_P7_NAP */
147	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
148				 NOTEST, 0x100)
149
150	. = 0x200
151machine_check_pSeries_1:
152	/* This is moved out of line as it can be patched by FW, but
153	 * some code path might still want to branch into the original
154	 * vector
155	 */
156	b	machine_check_pSeries
157
158	. = 0x300
159	.globl data_access_pSeries
160data_access_pSeries:
161	HMT_MEDIUM
162	SET_SCRATCH0(r13)
163BEGIN_FTR_SECTION
164	b	data_access_check_stab
165data_access_not_stab:
166END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
167	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
168				 KVMTEST, 0x300)
169
170	. = 0x380
171	.globl data_access_slb_pSeries
172data_access_slb_pSeries:
173	HMT_MEDIUM
174	SET_SCRATCH0(r13)
175	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
176	std	r3,PACA_EXSLB+EX_R3(r13)
177	mfspr	r3,SPRN_DAR
178#ifdef __DISABLED__
179	/* Keep that around for when we re-implement dynamic VSIDs */
180	cmpdi	r3,0
181	bge	slb_miss_user_pseries
182#endif /* __DISABLED__ */
183	mfspr	r12,SPRN_SRR1
184#ifndef CONFIG_RELOCATABLE
185	b	.slb_miss_realmode
186#else
187	/*
188	 * We can't just use a direct branch to .slb_miss_realmode
189	 * because the distance from here to there depends on where
190	 * the kernel ends up being put.
191	 */
192	mfctr	r11
193	ld	r10,PACAKBASE(r13)
194	LOAD_HANDLER(r10, .slb_miss_realmode)
195	mtctr	r10
196	bctr
197#endif
198
199	STD_EXCEPTION_PSERIES(0x400, 0x400, instruction_access)
200
201	. = 0x480
202	.globl instruction_access_slb_pSeries
203instruction_access_slb_pSeries:
204	HMT_MEDIUM
205	SET_SCRATCH0(r13)
206	EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
207	std	r3,PACA_EXSLB+EX_R3(r13)
208	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
209#ifdef __DISABLED__
210	/* Keep that around for when we re-implement dynamic VSIDs */
211	cmpdi	r3,0
212	bge	slb_miss_user_pseries
213#endif /* __DISABLED__ */
214	mfspr	r12,SPRN_SRR1
215#ifndef CONFIG_RELOCATABLE
216	b	.slb_miss_realmode
217#else
218	mfctr	r11
219	ld	r10,PACAKBASE(r13)
220	LOAD_HANDLER(r10, .slb_miss_realmode)
221	mtctr	r10
222	bctr
223#endif
224
225	/* We open code these as we can't have a ". = x" (even with
226	 * x = "." within a feature section
227	 */
228	. = 0x500;
229	.globl hardware_interrupt_pSeries;
230	.globl hardware_interrupt_hv;
231hardware_interrupt_pSeries:
232hardware_interrupt_hv:
233	BEGIN_FTR_SECTION
234		_MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
235					    EXC_HV, SOFTEN_TEST_HV)
236		KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
237	FTR_SECTION_ELSE
238		_MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
239					    EXC_STD, SOFTEN_TEST_HV_201)
240		KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
241	ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
242
243	STD_EXCEPTION_PSERIES(0x600, 0x600, alignment)
244	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x600)
245
246	STD_EXCEPTION_PSERIES(0x700, 0x700, program_check)
247	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x700)
248
249	STD_EXCEPTION_PSERIES(0x800, 0x800, fp_unavailable)
250	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x800)
251
252	MASKABLE_EXCEPTION_PSERIES(0x900, 0x900, decrementer)
253	STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
254
255	STD_EXCEPTION_PSERIES(0xa00, 0xa00, trap_0a)
256	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xa00)
257
258	STD_EXCEPTION_PSERIES(0xb00, 0xb00, trap_0b)
259	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xb00)
260
261	. = 0xc00
262	.globl	system_call_pSeries
263system_call_pSeries:
264	HMT_MEDIUM
265#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
266	SET_SCRATCH0(r13)
267	GET_PACA(r13)
268	std	r9,PACA_EXGEN+EX_R9(r13)
269	std	r10,PACA_EXGEN+EX_R10(r13)
270	mfcr	r9
271	KVMTEST(0xc00)
272	GET_SCRATCH0(r13)
273#endif
274	SYSCALL_PSERIES_1
275	SYSCALL_PSERIES_2_RFID
276	SYSCALL_PSERIES_3
277	KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
278
279	STD_EXCEPTION_PSERIES(0xd00, 0xd00, single_step)
280	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xd00)
281
282	/* At 0xe??? we have a bunch of hypervisor exceptions, we branch
283	 * out of line to handle them
284	 */
285	. = 0xe00
286hv_exception_trampoline:
287	b	h_data_storage_hv
288	. = 0xe20
289	b	h_instr_storage_hv
290	. = 0xe40
291	b	emulation_assist_hv
292	. = 0xe50
293	b	hmi_exception_hv
294	. = 0xe60
295	b	hmi_exception_hv
296
297	/* We need to deal with the Altivec unavailable exception
298	 * here which is at 0xf20, thus in the middle of the
299	 * prolog code of the PerformanceMonitor one. A little
300	 * trickery is thus necessary
301	 */
302performance_monitor_pSeries_1:
303	. = 0xf00
304	b	performance_monitor_pSeries
305
306altivec_unavailable_pSeries_1:
307	. = 0xf20
308	b	altivec_unavailable_pSeries
309
310vsx_unavailable_pSeries_1:
311	. = 0xf40
312	b	vsx_unavailable_pSeries
313
314#ifdef CONFIG_CBE_RAS
315	STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
316	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
317#endif /* CONFIG_CBE_RAS */
318
319	STD_EXCEPTION_PSERIES(0x1300, 0x1300, instruction_breakpoint)
320	KVM_HANDLER_PR_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
321
322	. = 0x1500
323	.global denorm_exception_hv
324denorm_exception_hv:
325	HMT_MEDIUM
326	mtspr	SPRN_SPRG_HSCRATCH0,r13
327	mfspr	r13,SPRN_SPRG_HPACA
328	std	r9,PACA_EXGEN+EX_R9(r13)
329	std	r10,PACA_EXGEN+EX_R10(r13)
330	std	r11,PACA_EXGEN+EX_R11(r13)
331	std	r12,PACA_EXGEN+EX_R12(r13)
332	mfspr	r9,SPRN_SPRG_HSCRATCH0
333	std	r9,PACA_EXGEN+EX_R13(r13)
334	mfcr	r9
335
336#ifdef CONFIG_PPC_DENORMALISATION
337	mfspr	r10,SPRN_HSRR1
338	mfspr	r11,SPRN_HSRR0		/* save HSRR0 */
339	andis.	r10,r10,(HSRR1_DENORM)@h /* denorm? */
340	addi	r11,r11,-4		/* HSRR0 is next instruction */
341	bne+	denorm_assist
342#endif
343
344	EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
345	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
346
347#ifdef CONFIG_CBE_RAS
348	STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
349	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
350#endif /* CONFIG_CBE_RAS */
351
352	STD_EXCEPTION_PSERIES(0x1700, 0x1700, altivec_assist)
353	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x1700)
354
355#ifdef CONFIG_CBE_RAS
356	STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
357	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
358#else
359	. = 0x1800
360#endif /* CONFIG_CBE_RAS */
361
362
363/*** Out of line interrupts support ***/
364
365	.align	7
366	/* moved from 0x200 */
367machine_check_pSeries:
368	.globl machine_check_fwnmi
369machine_check_fwnmi:
370	HMT_MEDIUM
371	SET_SCRATCH0(r13)		/* save r13 */
372	EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common,
373				 EXC_STD, KVMTEST, 0x200)
374	KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
375
376	/* moved from 0x300 */
377data_access_check_stab:
378	GET_PACA(r13)
379	std	r9,PACA_EXSLB+EX_R9(r13)
380	std	r10,PACA_EXSLB+EX_R10(r13)
381	mfspr	r10,SPRN_DAR
382	mfspr	r9,SPRN_DSISR
383	srdi	r10,r10,60
384	rlwimi	r10,r9,16,0x20
385#ifdef CONFIG_KVM_BOOK3S_PR
386	lbz	r9,HSTATE_IN_GUEST(r13)
387	rlwimi	r10,r9,8,0x300
388#endif
389	mfcr	r9
390	cmpwi	r10,0x2c
391	beq	do_stab_bolted_pSeries
392	mtcrf	0x80,r9
393	ld	r9,PACA_EXSLB+EX_R9(r13)
394	ld	r10,PACA_EXSLB+EX_R10(r13)
395	b	data_access_not_stab
396do_stab_bolted_pSeries:
397	std	r11,PACA_EXSLB+EX_R11(r13)
398	std	r12,PACA_EXSLB+EX_R12(r13)
399	GET_SCRATCH0(r10)
400	std	r10,PACA_EXSLB+EX_R13(r13)
401	EXCEPTION_PROLOG_PSERIES_1(.do_stab_bolted, EXC_STD)
402
403	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
404	KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
405	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x400)
406	KVM_HANDLER_PR(PACA_EXSLB, EXC_STD, 0x480)
407	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0x900)
408	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
409
410#ifdef CONFIG_PPC_DENORMALISATION
411denorm_assist:
412BEGIN_FTR_SECTION
413/*
414 * To denormalise we need to move a copy of the register to itself.
415 * For POWER6 do that here for all FP regs.
416 */
417	mfmsr	r10
418	ori	r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
419	xori	r10,r10,(MSR_FE0|MSR_FE1)
420	mtmsrd	r10
421	sync
422	fmr	0,0
423	fmr	1,1
424	fmr	2,2
425	fmr	3,3
426	fmr	4,4
427	fmr	5,5
428	fmr	6,6
429	fmr	7,7
430	fmr	8,8
431	fmr	9,9
432	fmr	10,10
433	fmr	11,11
434	fmr	12,12
435	fmr	13,13
436	fmr	14,14
437	fmr	15,15
438	fmr	16,16
439	fmr	17,17
440	fmr	18,18
441	fmr	19,19
442	fmr	20,20
443	fmr	21,21
444	fmr	22,22
445	fmr	23,23
446	fmr	24,24
447	fmr	25,25
448	fmr	26,26
449	fmr	27,27
450	fmr	28,28
451	fmr	29,29
452	fmr	30,30
453	fmr	31,31
454FTR_SECTION_ELSE
455/*
456 * To denormalise we need to move a copy of the register to itself.
457 * For POWER7 do that here for the first 32 VSX registers only.
458 */
459	mfmsr	r10
460	oris	r10,r10,MSR_VSX@h
461	mtmsrd	r10
462	sync
463	XVCPSGNDP(0,0,0)
464	XVCPSGNDP(1,1,1)
465	XVCPSGNDP(2,2,2)
466	XVCPSGNDP(3,3,3)
467	XVCPSGNDP(4,4,4)
468	XVCPSGNDP(5,5,5)
469	XVCPSGNDP(6,6,6)
470	XVCPSGNDP(7,7,7)
471	XVCPSGNDP(8,8,8)
472	XVCPSGNDP(9,9,9)
473	XVCPSGNDP(10,10,10)
474	XVCPSGNDP(11,11,11)
475	XVCPSGNDP(12,12,12)
476	XVCPSGNDP(13,13,13)
477	XVCPSGNDP(14,14,14)
478	XVCPSGNDP(15,15,15)
479	XVCPSGNDP(16,16,16)
480	XVCPSGNDP(17,17,17)
481	XVCPSGNDP(18,18,18)
482	XVCPSGNDP(19,19,19)
483	XVCPSGNDP(20,20,20)
484	XVCPSGNDP(21,21,21)
485	XVCPSGNDP(22,22,22)
486	XVCPSGNDP(23,23,23)
487	XVCPSGNDP(24,24,24)
488	XVCPSGNDP(25,25,25)
489	XVCPSGNDP(26,26,26)
490	XVCPSGNDP(27,27,27)
491	XVCPSGNDP(28,28,28)
492	XVCPSGNDP(29,29,29)
493	XVCPSGNDP(30,30,30)
494	XVCPSGNDP(31,31,31)
495ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
496	mtspr	SPRN_HSRR0,r11
497	mtcrf	0x80,r9
498	ld	r9,PACA_EXGEN+EX_R9(r13)
499	ld	r10,PACA_EXGEN+EX_R10(r13)
500	ld	r11,PACA_EXGEN+EX_R11(r13)
501	ld	r12,PACA_EXGEN+EX_R12(r13)
502	ld	r13,PACA_EXGEN+EX_R13(r13)
503	HRFID
504	b	.
505#endif
506
507	.align	7
508	/* moved from 0xe00 */
509	STD_EXCEPTION_HV(., 0xe02, h_data_storage)
510	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
511	STD_EXCEPTION_HV(., 0xe22, h_instr_storage)
512	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
513	STD_EXCEPTION_HV(., 0xe42, emulation_assist)
514	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
515	STD_EXCEPTION_HV(., 0xe62, hmi_exception) /* need to flush cache ? */
516	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
517
518	/* moved from 0xf00 */
519	STD_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
520	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf00)
521	STD_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
522	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf20)
523	STD_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
524	KVM_HANDLER_PR(PACA_EXGEN, EXC_STD, 0xf40)
525
526/*
527 * An interrupt came in while soft-disabled. We set paca->irq_happened,
528 * then, if it was a decrementer interrupt, we bump the dec to max and
529 * and return, else we hard disable and return. This is called with
530 * r10 containing the value to OR to the paca field.
531 */
532#define MASKED_INTERRUPT(_H)				\
533masked_##_H##interrupt:					\
534	std	r11,PACA_EXGEN+EX_R11(r13);		\
535	lbz	r11,PACAIRQHAPPENED(r13);		\
536	or	r11,r11,r10;				\
537	stb	r11,PACAIRQHAPPENED(r13);		\
538	andi.	r10,r10,PACA_IRQ_DEC;			\
539	beq	1f;					\
540	lis	r10,0x7fff;				\
541	ori	r10,r10,0xffff;				\
542	mtspr	SPRN_DEC,r10;				\
543	b	2f;					\
5441:	mfspr	r10,SPRN_##_H##SRR1;			\
545	rldicl	r10,r10,48,1; /* clear MSR_EE */	\
546	rotldi	r10,r10,16;				\
547	mtspr	SPRN_##_H##SRR1,r10;			\
5482:	mtcrf	0x80,r9;				\
549	ld	r9,PACA_EXGEN+EX_R9(r13);		\
550	ld	r10,PACA_EXGEN+EX_R10(r13);		\
551	ld	r11,PACA_EXGEN+EX_R11(r13);		\
552	GET_SCRATCH0(r13);				\
553	##_H##rfid;					\
554	b	.
555
556	MASKED_INTERRUPT()
557	MASKED_INTERRUPT(H)
558
559/*
560 * Called from arch_local_irq_enable when an interrupt needs
561 * to be resent. r3 contains 0x500 or 0x900 to indicate which
562 * kind of interrupt. MSR:EE is already off. We generate a
563 * stackframe like if a real interrupt had happened.
564 *
565 * Note: While MSR:EE is off, we need to make sure that _MSR
566 * in the generated frame has EE set to 1 or the exception
567 * handler will not properly re-enable them.
568 */
569_GLOBAL(__replay_interrupt)
570	/* We are going to jump to the exception common code which
571	 * will retrieve various register values from the PACA which
572	 * we don't give a damn about, so we don't bother storing them.
573	 */
574	mfmsr	r12
575	mflr	r11
576	mfcr	r9
577	ori	r12,r12,MSR_EE
578	andi.	r3,r3,0x0800
579	bne	decrementer_common
580	b	hardware_interrupt_common
581
582#ifdef CONFIG_PPC_PSERIES
583/*
584 * Vectors for the FWNMI option.  Share common code.
585 */
586	.globl system_reset_fwnmi
587      .align 7
588system_reset_fwnmi:
589	HMT_MEDIUM
590	SET_SCRATCH0(r13)		/* save r13 */
591	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
592				 NOTEST, 0x100)
593
594#endif /* CONFIG_PPC_PSERIES */
595
596#ifdef __DISABLED__
597/*
598 * This is used for when the SLB miss handler has to go virtual,
599 * which doesn't happen for now anymore but will once we re-implement
600 * dynamic VSIDs for shared page tables
601 */
602slb_miss_user_pseries:
603	std	r10,PACA_EXGEN+EX_R10(r13)
604	std	r11,PACA_EXGEN+EX_R11(r13)
605	std	r12,PACA_EXGEN+EX_R12(r13)
606	GET_SCRATCH0(r10)
607	ld	r11,PACA_EXSLB+EX_R9(r13)
608	ld	r12,PACA_EXSLB+EX_R3(r13)
609	std	r10,PACA_EXGEN+EX_R13(r13)
610	std	r11,PACA_EXGEN+EX_R9(r13)
611	std	r12,PACA_EXGEN+EX_R3(r13)
612	clrrdi	r12,r13,32
613	mfmsr	r10
614	mfspr	r11,SRR0			/* save SRR0 */
615	ori	r12,r12,slb_miss_user_common@l	/* virt addr of handler */
616	ori	r10,r10,MSR_IR|MSR_DR|MSR_RI
617	mtspr	SRR0,r12
618	mfspr	r12,SRR1			/* and SRR1 */
619	mtspr	SRR1,r10
620	rfid
621	b	.				/* prevent spec. execution */
622#endif /* __DISABLED__ */
623
624/*
625 * Code from here down to __end_handlers is invoked from the
626 * exception prologs above.  Because the prologs assemble the
627 * addresses of these handlers using the LOAD_HANDLER macro,
628 * which uses an ori instruction, these handlers must be in
629 * the first 64k of the kernel image.
630 */
631
632/*** Common interrupt handlers ***/
633
634	STD_EXCEPTION_COMMON(0x100, system_reset, .system_reset_exception)
635
636	/*
637	 * Machine check is different because we use a different
638	 * save area: PACA_EXMC instead of PACA_EXGEN.
639	 */
640	.align	7
641	.globl machine_check_common
642machine_check_common:
643	EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
644	FINISH_NAP
645	DISABLE_INTS
646	bl	.save_nvgprs
647	addi	r3,r1,STACK_FRAME_OVERHEAD
648	bl	.machine_check_exception
649	b	.ret_from_except
650
651	STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
652	STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, .timer_interrupt)
653	STD_EXCEPTION_COMMON(0x980, hdecrementer, .hdec_interrupt)
654	STD_EXCEPTION_COMMON(0xa00, trap_0a, .unknown_exception)
655	STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
656	STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
657	STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
658	STD_EXCEPTION_COMMON(0xe40, emulation_assist, .program_check_exception)
659	STD_EXCEPTION_COMMON(0xe60, hmi_exception, .unknown_exception)
660	STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, .performance_monitor_exception)
661	STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
662	STD_EXCEPTION_COMMON(0x1502, denorm, .unknown_exception)
663#ifdef CONFIG_ALTIVEC
664	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
665#else
666	STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
667#endif
668#ifdef CONFIG_CBE_RAS
669	STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
670	STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
671	STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
672#endif /* CONFIG_CBE_RAS */
673
674	/*
675	 * Relocation-on interrupts: A subset of the interrupts can be delivered
676	 * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
677	 * it.  Addresses are the same as the original interrupt addresses, but
678	 * offset by 0xc000000000004000.
679	 * It's impossible to receive interrupts below 0x300 via this mechanism.
680	 * KVM: None of these traps are from the guest ; anything that escalated
681	 * to HV=1 from HV=0 is delivered via real mode handlers.
682	 */
683
684	/*
685	 * This uses the standard macro, since the original 0x300 vector
686	 * only has extra guff for STAB-based processors -- which never
687	 * come here.
688	 */
689	STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
690	. = 0x4380
691	.globl data_access_slb_relon_pSeries
692data_access_slb_relon_pSeries:
693	HMT_MEDIUM
694	SET_SCRATCH0(r13)
695	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
696	std	r3,PACA_EXSLB+EX_R3(r13)
697	mfspr	r3,SPRN_DAR
698	mfspr	r12,SPRN_SRR1
699#ifndef CONFIG_RELOCATABLE
700	b	.slb_miss_realmode
701#else
702	/*
703	 * We can't just use a direct branch to .slb_miss_realmode
704	 * because the distance from here to there depends on where
705	 * the kernel ends up being put.
706	 */
707	mfctr	r11
708	ld	r10,PACAKBASE(r13)
709	LOAD_HANDLER(r10, .slb_miss_realmode)
710	mtctr	r10
711	bctr
712#endif
713
714	STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
715	. = 0x4480
716	.globl instruction_access_slb_relon_pSeries
717instruction_access_slb_relon_pSeries:
718	HMT_MEDIUM
719	SET_SCRATCH0(r13)
720	EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
721	std	r3,PACA_EXSLB+EX_R3(r13)
722	mfspr	r3,SPRN_SRR0		/* SRR0 is faulting address */
723	mfspr	r12,SPRN_SRR1
724#ifndef CONFIG_RELOCATABLE
725	b	.slb_miss_realmode
726#else
727	mfctr	r11
728	ld	r10,PACAKBASE(r13)
729	LOAD_HANDLER(r10, .slb_miss_realmode)
730	mtctr	r10
731	bctr
732#endif
733
734	. = 0x4500
735	.globl hardware_interrupt_relon_pSeries;
736	.globl hardware_interrupt_relon_hv;
737hardware_interrupt_relon_pSeries:
738hardware_interrupt_relon_hv:
739	BEGIN_FTR_SECTION
740		_MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
741	FTR_SECTION_ELSE
742		_MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
743	ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_206)
744	STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
745	STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
746	STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
747	MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
748	STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
749	STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
750
751	. = 0x4c00
752	.globl system_call_relon_pSeries
753system_call_relon_pSeries:
754	HMT_MEDIUM
755	SYSCALL_PSERIES_1
756	SYSCALL_PSERIES_2_DIRECT
757	SYSCALL_PSERIES_3
758
759	STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
760
761	. = 0x4e00
762	b	h_data_storage_relon_hv
763
764	. = 0x4e20
765	b	h_instr_storage_relon_hv
766
767	. = 0x4e40
768	b	emulation_assist_relon_hv
769
770	. = 0x4e50
771	b	hmi_exception_relon_hv
772
773	. = 0x4e60
774	b	hmi_exception_relon_hv
775
776	/* For when we support the doorbell interrupt:
777	STD_RELON_EXCEPTION_HYPERVISOR(0x4e80, 0xe80, doorbell_hyper)
778	*/
779
780performance_monitor_relon_pSeries_1:
781	. = 0x4f00
782	b	performance_monitor_relon_pSeries
783
784altivec_unavailable_relon_pSeries_1:
785	. = 0x4f20
786	b	altivec_unavailable_relon_pSeries
787
788vsx_unavailable_relon_pSeries_1:
789	. = 0x4f40
790	b	vsx_unavailable_relon_pSeries
791
792#ifdef CONFIG_CBE_RAS
793	STD_RELON_EXCEPTION_HV(0x5200, 0x1202, cbe_system_error)
794#endif /* CONFIG_CBE_RAS */
795	STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
796#ifdef CONFIG_PPC_DENORMALISATION
797	. = 0x5500
798	b	denorm_exception_hv
799#endif
800#ifdef CONFIG_CBE_RAS
801	STD_RELON_EXCEPTION_HV(0x5600, 0x1602, cbe_maintenance)
802#else
803#ifdef CONFIG_HVC_SCOM
804	STD_RELON_EXCEPTION_HV(0x5600, 0x1600, maintence_interrupt)
805	KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1600)
806#endif /* CONFIG_HVC_SCOM */
807#endif /* CONFIG_CBE_RAS */
808	STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
809#ifdef CONFIG_CBE_RAS
810	STD_RELON_EXCEPTION_HV(0x5800, 0x1802, cbe_thermal)
811#endif /* CONFIG_CBE_RAS */
812
813	/* Other future vectors */
814	.align	7
815	.globl	__end_interrupts
816__end_interrupts:
817
818	.align	7
819system_call_entry_direct:
820#if defined(CONFIG_RELOCATABLE)
821	/* The first level prologue may have used LR to get here, saving
822	 * orig in r10.  To save hacking/ifdeffing common code, restore here.
823	 */
824	mtlr	r10
825#endif
826system_call_entry:
827	b	system_call_common
828
829ppc64_runlatch_on_trampoline:
830	b	.__ppc64_runlatch_on
831
832/*
833 * Here we have detected that the kernel stack pointer is bad.
834 * R9 contains the saved CR, r13 points to the paca,
835 * r10 contains the (bad) kernel stack pointer,
836 * r11 and r12 contain the saved SRR0 and SRR1.
837 * We switch to using an emergency stack, save the registers there,
838 * and call kernel_bad_stack(), which panics.
839 */
840bad_stack:
841	ld	r1,PACAEMERGSP(r13)
842	subi	r1,r1,64+INT_FRAME_SIZE
843	std	r9,_CCR(r1)
844	std	r10,GPR1(r1)
845	std	r11,_NIP(r1)
846	std	r12,_MSR(r1)
847	mfspr	r11,SPRN_DAR
848	mfspr	r12,SPRN_DSISR
849	std	r11,_DAR(r1)
850	std	r12,_DSISR(r1)
851	mflr	r10
852	mfctr	r11
853	mfxer	r12
854	std	r10,_LINK(r1)
855	std	r11,_CTR(r1)
856	std	r12,_XER(r1)
857	SAVE_GPR(0,r1)
858	SAVE_GPR(2,r1)
859	ld	r10,EX_R3(r3)
860	std	r10,GPR3(r1)
861	SAVE_GPR(4,r1)
862	SAVE_4GPRS(5,r1)
863	ld	r9,EX_R9(r3)
864	ld	r10,EX_R10(r3)
865	SAVE_2GPRS(9,r1)
866	ld	r9,EX_R11(r3)
867	ld	r10,EX_R12(r3)
868	ld	r11,EX_R13(r3)
869	std	r9,GPR11(r1)
870	std	r10,GPR12(r1)
871	std	r11,GPR13(r1)
872BEGIN_FTR_SECTION
873	ld	r10,EX_CFAR(r3)
874	std	r10,ORIG_GPR3(r1)
875END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
876	SAVE_8GPRS(14,r1)
877	SAVE_10GPRS(22,r1)
878	lhz	r12,PACA_TRAP_SAVE(r13)
879	std	r12,_TRAP(r1)
880	addi	r11,r1,INT_FRAME_SIZE
881	std	r11,0(r1)
882	li	r12,0
883	std	r12,0(r11)
884	ld	r2,PACATOC(r13)
885	ld	r11,exception_marker@toc(r2)
886	std	r12,RESULT(r1)
887	std	r11,STACK_FRAME_OVERHEAD-16(r1)
8881:	addi	r3,r1,STACK_FRAME_OVERHEAD
889	bl	.kernel_bad_stack
890	b	1b
891
892/*
893 * Here r13 points to the paca, r9 contains the saved CR,
894 * SRR0 and SRR1 are saved in r11 and r12,
895 * r9 - r13 are saved in paca->exgen.
896 */
897	.align	7
898	.globl data_access_common
899data_access_common:
900	mfspr	r10,SPRN_DAR
901	std	r10,PACA_EXGEN+EX_DAR(r13)
902	mfspr	r10,SPRN_DSISR
903	stw	r10,PACA_EXGEN+EX_DSISR(r13)
904	EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
905	DISABLE_INTS
906	ld	r12,_MSR(r1)
907	ld	r3,PACA_EXGEN+EX_DAR(r13)
908	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
909	li	r5,0x300
910	b	.do_hash_page		/* Try to handle as hpte fault */
911
912	.align  7
913	.globl  h_data_storage_common
914h_data_storage_common:
915	mfspr   r10,SPRN_HDAR
916	std     r10,PACA_EXGEN+EX_DAR(r13)
917	mfspr   r10,SPRN_HDSISR
918	stw     r10,PACA_EXGEN+EX_DSISR(r13)
919	EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
920	bl      .save_nvgprs
921	DISABLE_INTS
922	addi    r3,r1,STACK_FRAME_OVERHEAD
923	bl      .unknown_exception
924	b       .ret_from_except
925
926	.align	7
927	.globl instruction_access_common
928instruction_access_common:
929	EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
930	DISABLE_INTS
931	ld	r12,_MSR(r1)
932	ld	r3,_NIP(r1)
933	andis.	r4,r12,0x5820
934	li	r5,0x400
935	b	.do_hash_page		/* Try to handle as hpte fault */
936
937	STD_EXCEPTION_COMMON(0xe20, h_instr_storage, .unknown_exception)
938
939/*
940 * Here is the common SLB miss user that is used when going to virtual
941 * mode for SLB misses, that is currently not used
942 */
943#ifdef __DISABLED__
944	.align	7
945	.globl	slb_miss_user_common
946slb_miss_user_common:
947	mflr	r10
948	std	r3,PACA_EXGEN+EX_DAR(r13)
949	stw	r9,PACA_EXGEN+EX_CCR(r13)
950	std	r10,PACA_EXGEN+EX_LR(r13)
951	std	r11,PACA_EXGEN+EX_SRR0(r13)
952	bl	.slb_allocate_user
953
954	ld	r10,PACA_EXGEN+EX_LR(r13)
955	ld	r3,PACA_EXGEN+EX_R3(r13)
956	lwz	r9,PACA_EXGEN+EX_CCR(r13)
957	ld	r11,PACA_EXGEN+EX_SRR0(r13)
958	mtlr	r10
959	beq-	slb_miss_fault
960
961	andi.	r10,r12,MSR_RI		/* check for unrecoverable exception */
962	beq-	unrecov_user_slb
963	mfmsr	r10
964
965.machine push
966.machine "power4"
967	mtcrf	0x80,r9
968.machine pop
969
970	clrrdi	r10,r10,2		/* clear RI before setting SRR0/1 */
971	mtmsrd	r10,1
972
973	mtspr	SRR0,r11
974	mtspr	SRR1,r12
975
976	ld	r9,PACA_EXGEN+EX_R9(r13)
977	ld	r10,PACA_EXGEN+EX_R10(r13)
978	ld	r11,PACA_EXGEN+EX_R11(r13)
979	ld	r12,PACA_EXGEN+EX_R12(r13)
980	ld	r13,PACA_EXGEN+EX_R13(r13)
981	rfid
982	b	.
983
984slb_miss_fault:
985	EXCEPTION_PROLOG_COMMON(0x380, PACA_EXGEN)
986	ld	r4,PACA_EXGEN+EX_DAR(r13)
987	li	r5,0
988	std	r4,_DAR(r1)
989	std	r5,_DSISR(r1)
990	b	handle_page_fault
991
992unrecov_user_slb:
993	EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
994	DISABLE_INTS
995	bl	.save_nvgprs
9961:	addi	r3,r1,STACK_FRAME_OVERHEAD
997	bl	.unrecoverable_exception
998	b	1b
999
1000#endif /* __DISABLED__ */
1001
1002
1003/*
1004 * r13 points to the PACA, r9 contains the saved CR,
1005 * r12 contain the saved SRR1, SRR0 is still ready for return
1006 * r3 has the faulting address
1007 * r9 - r13 are saved in paca->exslb.
1008 * r3 is saved in paca->slb_r3
1009 * We assume we aren't going to take any exceptions during this procedure.
1010 */
1011_GLOBAL(slb_miss_realmode)
1012	mflr	r10
1013#ifdef CONFIG_RELOCATABLE
1014	mtctr	r11
1015#endif
1016
1017	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
1018	std	r10,PACA_EXSLB+EX_LR(r13)	/* save LR */
1019
1020	bl	.slb_allocate_realmode
1021
1022	/* All done -- return from exception. */
1023
1024	ld	r10,PACA_EXSLB+EX_LR(r13)
1025	ld	r3,PACA_EXSLB+EX_R3(r13)
1026	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
1027
1028	mtlr	r10
1029
1030	andi.	r10,r12,MSR_RI	/* check for unrecoverable exception */
1031	beq-	2f
1032
1033.machine	push
1034.machine	"power4"
1035	mtcrf	0x80,r9
1036	mtcrf	0x01,r9		/* slb_allocate uses cr0 and cr7 */
1037.machine	pop
1038
1039	ld	r9,PACA_EXSLB+EX_R9(r13)
1040	ld	r10,PACA_EXSLB+EX_R10(r13)
1041	ld	r11,PACA_EXSLB+EX_R11(r13)
1042	ld	r12,PACA_EXSLB+EX_R12(r13)
1043	ld	r13,PACA_EXSLB+EX_R13(r13)
1044	rfid
1045	b	.	/* prevent speculative execution */
1046
10472:	mfspr	r11,SPRN_SRR0
1048	ld	r10,PACAKBASE(r13)
1049	LOAD_HANDLER(r10,unrecov_slb)
1050	mtspr	SPRN_SRR0,r10
1051	ld	r10,PACAKMSR(r13)
1052	mtspr	SPRN_SRR1,r10
1053	rfid
1054	b	.
1055
1056unrecov_slb:
1057	EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
1058	DISABLE_INTS
1059	bl	.save_nvgprs
10601:	addi	r3,r1,STACK_FRAME_OVERHEAD
1061	bl	.unrecoverable_exception
1062	b	1b
1063
1064
1065#ifdef CONFIG_PPC_970_NAP
1066power4_fixup_nap:
1067	andc	r9,r9,r10
1068	std	r9,TI_LOCAL_FLAGS(r11)
1069	ld	r10,_LINK(r1)		/* make idle task do the */
1070	std	r10,_NIP(r1)		/* equivalent of a blr */
1071	blr
1072#endif
1073
1074	.align	7
1075	.globl alignment_common
1076alignment_common:
1077	mfspr	r10,SPRN_DAR
1078	std	r10,PACA_EXGEN+EX_DAR(r13)
1079	mfspr	r10,SPRN_DSISR
1080	stw	r10,PACA_EXGEN+EX_DSISR(r13)
1081	EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN)
1082	ld	r3,PACA_EXGEN+EX_DAR(r13)
1083	lwz	r4,PACA_EXGEN+EX_DSISR(r13)
1084	std	r3,_DAR(r1)
1085	std	r4,_DSISR(r1)
1086	bl	.save_nvgprs
1087	DISABLE_INTS
1088	addi	r3,r1,STACK_FRAME_OVERHEAD
1089	bl	.alignment_exception
1090	b	.ret_from_except
1091
1092	.align	7
1093	.globl program_check_common
1094program_check_common:
1095	EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
1096	bl	.save_nvgprs
1097	DISABLE_INTS
1098	addi	r3,r1,STACK_FRAME_OVERHEAD
1099	bl	.program_check_exception
1100	b	.ret_from_except
1101
1102	.align	7
1103	.globl fp_unavailable_common
1104fp_unavailable_common:
1105	EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
1106	bne	1f			/* if from user, just load it up */
1107	bl	.save_nvgprs
1108	DISABLE_INTS
1109	addi	r3,r1,STACK_FRAME_OVERHEAD
1110	bl	.kernel_fp_unavailable_exception
1111	BUG_OPCODE
11121:	bl	.load_up_fpu
1113	b	fast_exception_return
1114
1115	.align	7
1116	.globl altivec_unavailable_common
1117altivec_unavailable_common:
1118	EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
1119#ifdef CONFIG_ALTIVEC
1120BEGIN_FTR_SECTION
1121	beq	1f
1122	bl	.load_up_altivec
1123	b	fast_exception_return
11241:
1125END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
1126#endif
1127	bl	.save_nvgprs
1128	DISABLE_INTS
1129	addi	r3,r1,STACK_FRAME_OVERHEAD
1130	bl	.altivec_unavailable_exception
1131	b	.ret_from_except
1132
1133	.align	7
1134	.globl vsx_unavailable_common
1135vsx_unavailable_common:
1136	EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
1137#ifdef CONFIG_VSX
1138BEGIN_FTR_SECTION
1139	beq	1f
1140	b	.load_up_vsx
11411:
1142END_FTR_SECTION_IFSET(CPU_FTR_VSX)
1143#endif
1144	bl	.save_nvgprs
1145	DISABLE_INTS
1146	addi	r3,r1,STACK_FRAME_OVERHEAD
1147	bl	.vsx_unavailable_exception
1148	b	.ret_from_except
1149
1150	.align	7
1151	.globl	__end_handlers
1152__end_handlers:
1153
1154/*
1155 * Hash table stuff
1156 */
1157	.align	7
1158_STATIC(do_hash_page)
1159	std	r3,_DAR(r1)
1160	std	r4,_DSISR(r1)
1161
1162	andis.	r0,r4,0xa410		/* weird error? */
1163	bne-	handle_page_fault	/* if not, try to insert a HPTE */
1164	andis.  r0,r4,DSISR_DABRMATCH@h
1165	bne-    handle_dabr_fault
1166
1167BEGIN_FTR_SECTION
1168	andis.	r0,r4,0x0020		/* Is it a segment table fault? */
1169	bne-	do_ste_alloc		/* If so handle it */
1170END_MMU_FTR_SECTION_IFCLR(MMU_FTR_SLB)
1171
1172	CURRENT_THREAD_INFO(r11, r1)
1173	lwz	r0,TI_PREEMPT(r11)	/* If we're in an "NMI" */
1174	andis.	r0,r0,NMI_MASK@h	/* (i.e. an irq when soft-disabled) */
1175	bne	77f			/* then don't call hash_page now */
1176	/*
1177	 * We need to set the _PAGE_USER bit if MSR_PR is set or if we are
1178	 * accessing a userspace segment (even from the kernel). We assume
1179	 * kernel addresses always have the high bit set.
1180	 */
1181	rlwinm	r4,r4,32-25+9,31-9,31-9	/* DSISR_STORE -> _PAGE_RW */
1182	rotldi	r0,r3,15		/* Move high bit into MSR_PR posn */
1183	orc	r0,r12,r0		/* MSR_PR | ~high_bit */
1184	rlwimi	r4,r0,32-13,30,30	/* becomes _PAGE_USER access bit */
1185	ori	r4,r4,1			/* add _PAGE_PRESENT */
1186	rlwimi	r4,r5,22+2,31-2,31-2	/* Set _PAGE_EXEC if trap is 0x400 */
1187
1188	/*
1189	 * r3 contains the faulting address
1190	 * r4 contains the required access permissions
1191	 * r5 contains the trap number
1192	 *
1193	 * at return r3 = 0 for success, 1 for page fault, negative for error
1194	 */
1195	bl	.hash_page		/* build HPTE if possible */
1196	cmpdi	r3,0			/* see if hash_page succeeded */
1197
1198	/* Success */
1199	beq	fast_exc_return_irq	/* Return from exception on success */
1200
1201	/* Error */
1202	blt-	13f
1203
1204/* Here we have a page fault that hash_page can't handle. */
1205handle_page_fault:
120611:	ld	r4,_DAR(r1)
1207	ld	r5,_DSISR(r1)
1208	addi	r3,r1,STACK_FRAME_OVERHEAD
1209	bl	.do_page_fault
1210	cmpdi	r3,0
1211	beq+	12f
1212	bl	.save_nvgprs
1213	mr	r5,r3
1214	addi	r3,r1,STACK_FRAME_OVERHEAD
1215	lwz	r4,_DAR(r1)
1216	bl	.bad_page_fault
1217	b	.ret_from_except
1218
1219/* We have a data breakpoint exception - handle it */
1220handle_dabr_fault:
1221	bl	.save_nvgprs
1222	ld      r4,_DAR(r1)
1223	ld      r5,_DSISR(r1)
1224	addi    r3,r1,STACK_FRAME_OVERHEAD
1225	bl      .do_dabr
122612:	b       .ret_from_except_lite
1227
1228
1229/* We have a page fault that hash_page could handle but HV refused
1230 * the PTE insertion
1231 */
123213:	bl	.save_nvgprs
1233	mr	r5,r3
1234	addi	r3,r1,STACK_FRAME_OVERHEAD
1235	ld	r4,_DAR(r1)
1236	bl	.low_hash_fault
1237	b	.ret_from_except
1238
1239/*
1240 * We come here as a result of a DSI at a point where we don't want
1241 * to call hash_page, such as when we are accessing memory (possibly
1242 * user memory) inside a PMU interrupt that occurred while interrupts
1243 * were soft-disabled.  We want to invoke the exception handler for
1244 * the access, or panic if there isn't a handler.
1245 */
124677:	bl	.save_nvgprs
1247	mr	r4,r3
1248	addi	r3,r1,STACK_FRAME_OVERHEAD
1249	li	r5,SIGSEGV
1250	bl	.bad_page_fault
1251	b	.ret_from_except
1252
1253	/* here we have a segment miss */
1254do_ste_alloc:
1255	bl	.ste_allocate		/* try to insert stab entry */
1256	cmpdi	r3,0
1257	bne-	handle_page_fault
1258	b	fast_exception_return
1259
1260/*
1261 * r13 points to the PACA, r9 contains the saved CR,
1262 * r11 and r12 contain the saved SRR0 and SRR1.
1263 * r9 - r13 are saved in paca->exslb.
1264 * We assume we aren't going to take any exceptions during this procedure.
1265 * We assume (DAR >> 60) == 0xc.
1266 */
1267	.align	7
1268_GLOBAL(do_stab_bolted)
1269	stw	r9,PACA_EXSLB+EX_CCR(r13)	/* save CR in exc. frame */
1270	std	r11,PACA_EXSLB+EX_SRR0(r13)	/* save SRR0 in exc. frame */
1271
1272	/* Hash to the primary group */
1273	ld	r10,PACASTABVIRT(r13)
1274	mfspr	r11,SPRN_DAR
1275	srdi	r11,r11,28
1276	rldimi	r10,r11,7,52	/* r10 = first ste of the group */
1277
1278	/* Calculate VSID */
1279	/* This is a kernel address, so protovsid = ESID | 1 << 37 */
1280	li	r9,0x1
1281	rldimi  r11,r9,(CONTEXT_BITS + USER_ESID_BITS),0
1282	ASM_VSID_SCRAMBLE(r11, r9, 256M)
1283	rldic	r9,r11,12,16	/* r9 = vsid << 12 */
1284
1285	/* Search the primary group for a free entry */
12861:	ld	r11,0(r10)	/* Test valid bit of the current ste	*/
1287	andi.	r11,r11,0x80
1288	beq	2f
1289	addi	r10,r10,16
1290	andi.	r11,r10,0x70
1291	bne	1b
1292
1293	/* Stick for only searching the primary group for now.		*/
1294	/* At least for now, we use a very simple random castout scheme */
1295	/* Use the TB as a random number ;  OR in 1 to avoid entry 0	*/
1296	mftb	r11
1297	rldic	r11,r11,4,57	/* r11 = (r11 << 4) & 0x70 */
1298	ori	r11,r11,0x10
1299
1300	/* r10 currently points to an ste one past the group of interest */
1301	/* make it point to the randomly selected entry			*/
1302	subi	r10,r10,128
1303	or 	r10,r10,r11	/* r10 is the entry to invalidate	*/
1304
1305	isync			/* mark the entry invalid		*/
1306	ld	r11,0(r10)
1307	rldicl	r11,r11,56,1	/* clear the valid bit */
1308	rotldi	r11,r11,8
1309	std	r11,0(r10)
1310	sync
1311
1312	clrrdi	r11,r11,28	/* Get the esid part of the ste		*/
1313	slbie	r11
1314
13152:	std	r9,8(r10)	/* Store the vsid part of the ste	*/
1316	eieio
1317
1318	mfspr	r11,SPRN_DAR		/* Get the new esid			*/
1319	clrrdi	r11,r11,28	/* Permits a full 32b of ESID		*/
1320	ori	r11,r11,0x90	/* Turn on valid and kp			*/
1321	std	r11,0(r10)	/* Put new entry back into the stab	*/
1322
1323	sync
1324
1325	/* All done -- return from exception. */
1326	lwz	r9,PACA_EXSLB+EX_CCR(r13)	/* get saved CR */
1327	ld	r11,PACA_EXSLB+EX_SRR0(r13)	/* get saved SRR0 */
1328
1329	andi.	r10,r12,MSR_RI
1330	beq-	unrecov_slb
1331
1332	mtcrf	0x80,r9			/* restore CR */
1333
1334	mfmsr	r10
1335	clrrdi	r10,r10,2
1336	mtmsrd	r10,1
1337
1338	mtspr	SPRN_SRR0,r11
1339	mtspr	SPRN_SRR1,r12
1340	ld	r9,PACA_EXSLB+EX_R9(r13)
1341	ld	r10,PACA_EXSLB+EX_R10(r13)
1342	ld	r11,PACA_EXSLB+EX_R11(r13)
1343	ld	r12,PACA_EXSLB+EX_R12(r13)
1344	ld	r13,PACA_EXSLB+EX_R13(r13)
1345	rfid
1346	b	.	/* prevent speculative execution */
1347
1348
1349	/* Equivalents to the above handlers for relocation-on interrupt vectors */
1350	STD_RELON_EXCEPTION_HV(., 0xe00, h_data_storage)
1351	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe00)
1352	STD_RELON_EXCEPTION_HV(., 0xe20, h_instr_storage)
1353	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe20)
1354	STD_RELON_EXCEPTION_HV(., 0xe40, emulation_assist)
1355	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe40)
1356	STD_RELON_EXCEPTION_HV(., 0xe60, hmi_exception)
1357	KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe60)
1358
1359	STD_RELON_EXCEPTION_PSERIES(., 0xf00, performance_monitor)
1360	STD_RELON_EXCEPTION_PSERIES(., 0xf20, altivec_unavailable)
1361	STD_RELON_EXCEPTION_PSERIES(., 0xf40, vsx_unavailable)
1362
1363#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
1364/*
1365 * Data area reserved for FWNMI option.
1366 * This address (0x7000) is fixed by the RPA.
1367 */
1368	.= 0x7000
1369	.globl fwnmi_data_area
1370fwnmi_data_area:
1371
1372	/* pseries and powernv need to keep the whole page from
1373	 * 0x7000 to 0x8000 free for use by the firmware
1374	 */
1375	. = 0x8000
1376#endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
1377
1378/* Space for CPU0's segment table */
1379	.balign 4096
1380	.globl initial_stab
1381initial_stab:
1382	.space	4096
1383
1384#ifdef CONFIG_PPC_POWERNV
1385_GLOBAL(opal_mc_secondary_handler)
1386	HMT_MEDIUM
1387	SET_SCRATCH0(r13)
1388	GET_PACA(r13)
1389	clrldi	r3,r3,2
1390	tovirt(r3,r3)
1391	std	r3,PACA_OPAL_MC_EVT(r13)
1392	ld	r13,OPAL_MC_SRR0(r3)
1393	mtspr	SPRN_SRR0,r13
1394	ld	r13,OPAL_MC_SRR1(r3)
1395	mtspr	SPRN_SRR1,r13
1396	ld	r3,OPAL_MC_GPR3(r3)
1397	GET_SCRATCH0(r13)
1398	b	machine_check_pSeries
1399#endif /* CONFIG_PPC_POWERNV */
1400