xref: /linux/arch/mips/kernel/genex.S (revision ecb9194d1724d1265c8cfe50fcb1dad718476e5e)
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License.  See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8 * Copyright (C) 2002, 2007  Maciej W. Rozycki
9 * Copyright (C) 2001, 2012 MIPS Technologies, Inc.  All rights reserved.
10 */
11#include <linux/init.h>
12
13#include <asm/asm.h>
14#include <asm/asmmacro.h>
15#include <asm/cacheops.h>
16#include <asm/irqflags.h>
17#include <asm/regdef.h>
18#include <asm/fpregdef.h>
19#include <asm/mipsregs.h>
20#include <asm/stackframe.h>
21#include <asm/sync.h>
22#include <asm/thread_info.h>
23
24	__INIT
25
26/*
27 * General exception vector for all other CPUs.
28 *
29 * Be careful when changing this, it has to be at most 128 bytes
30 * to fit into space reserved for the exception handler.
31 */
32NESTED(except_vec3_generic, 0, sp)
33	.set	push
34	.set	noat
35	mfc0	k1, CP0_CAUSE
36	andi	k1, k1, 0x7c
37#ifdef CONFIG_64BIT
38	dsll	k1, k1, 1
39#endif
40	PTR_L	k0, exception_handlers(k1)
41	jr	k0
42	.set	pop
43	END(except_vec3_generic)
44
45/*
46 * General exception handler for CPUs with virtual coherency exception.
47 *
48 * Be careful when changing this, it has to be at most 256 (as a special
49 * exception) bytes to fit into space reserved for the exception handler.
50 */
51NESTED(except_vec3_r4000, 0, sp)
52	.set	push
53	.set	arch=r4000
54	.set	noat
55	mfc0	k1, CP0_CAUSE
56	li	k0, 31<<2
57	andi	k1, k1, 0x7c
58	.set	push
59	.set	noreorder
60	.set	nomacro
61	beq	k1, k0, handle_vced
62	 li	k0, 14<<2
63	beq	k1, k0, handle_vcei
64#ifdef CONFIG_64BIT
65	 dsll	k1, k1, 1
66#endif
67	.set	pop
68	PTR_L	k0, exception_handlers(k1)
69	jr	k0
70
71	/*
72	 * Big shit, we now may have two dirty primary cache lines for the same
73	 * physical address.  We can safely invalidate the line pointed to by
74	 * c0_badvaddr because after return from this exception handler the
75	 * load / store will be re-executed.
76	 */
77handle_vced:
78	MFC0	k0, CP0_BADVADDR
79	li	k1, -4					# Is this ...
80	and	k0, k1					# ... really needed?
81	mtc0	zero, CP0_TAGLO
82	cache	Index_Store_Tag_D, (k0)
83	cache	Hit_Writeback_Inv_SD, (k0)
84#ifdef CONFIG_PROC_FS
85	PTR_LA	k0, vced_count
86	lw	k1, (k0)
87	addiu	k1, 1
88	sw	k1, (k0)
89#endif
90	eret
91
92handle_vcei:
93	MFC0	k0, CP0_BADVADDR
94	cache	Hit_Writeback_Inv_SD, (k0)		# also cleans pi
95#ifdef CONFIG_PROC_FS
96	PTR_LA	k0, vcei_count
97	lw	k1, (k0)
98	addiu	k1, 1
99	sw	k1, (k0)
100#endif
101	eret
102	.set	pop
103	END(except_vec3_r4000)
104
105	__FINIT
106
107	.section .cpuidle.text,"ax"
108	/* Align to 32 bytes for the maximum idle interrupt region size. */
109	.align	5
110LEAF(r4k_wait)
111	/* Keep the ISA bit clear for calculations on local labels here. */
1120:	.fill 	0
113	/* Start of idle interrupt region. */
114	local_irq_enable
115	/*
116	 * If an interrupt lands here, before going idle on the next
117	 * instruction, we must *NOT* go idle since the interrupt could
118	 * have set TIF_NEED_RESCHED or caused a timer to need resched.
119	 * Fall through -- see skipover_handler below -- and have the
120	 * idle loop take care of things.
121	 */
1221:	.fill	0
123	/* The R2 EI/EHB sequence takes 8 bytes, otherwise pad up.  */
124	.if		1b - 0b > 32
125	.error	"overlong idle interrupt region"
126	.elseif	1b - 0b > 8
127	.align	4
128	.endif
1292:	.fill	0
130	.equ	r4k_wait_idle_size, 2b - 0b
131	/* End of idle interrupt region; size has to be a power of 2. */
132	.set	MIPS_ISA_ARCH_LEVEL_RAW
133r4k_wait_insn:
134	wait
135r4k_wait_exit:
136	.set	mips0
137	local_irq_disable
138	jr	ra
139	END(r4k_wait)
140	.previous
141
142	.macro	BUILD_SKIPOVER_PROLOGUE handler
143	FEXPORT(skipover_\handler)
144	.set	push
145	.set	noat
146	MFC0	k0, CP0_EPC
147	/* Subtract/add 2 to let the ISA bit propagate through the mask.  */
148	PTR_LA	k1, r4k_wait_insn - 2
149	ori 	k0, r4k_wait_idle_size - 2
150	.set	noreorder
151	bne	k0, k1, \handler
152	PTR_ADDIU 	k0, r4k_wait_exit - r4k_wait_insn + 2
153	.set	reorder
154	MTC0	k0, CP0_EPC
155	.set pop
156	.endm
157
158	.align	5
159BUILD_SKIPOVER_PROLOGUE handle_int
160NESTED(handle_int, PT_SIZE, sp)
161	.cfi_signal_frame
162#ifdef CONFIG_TRACE_IRQFLAGS
163	/*
164	 * Check to see if the interrupted code has just disabled
165	 * interrupts and ignore this interrupt for now if so.
166	 *
167	 * local_irq_disable() disables interrupts and then calls
168	 * trace_hardirqs_off() to track the state. If an interrupt is taken
169	 * after interrupts are disabled but before the state is updated
170	 * it will appear to restore_all that it is incorrectly returning with
171	 * interrupts disabled
172	 */
173	.set	push
174	.set	noat
175	mfc0	k0, CP0_STATUS
176#if defined(CONFIG_CPU_R3000)
177	and	k0, ST0_IEP
178	bnez	k0, 1f
179
180	mfc0	k0, CP0_EPC
181	.set	noreorder
182	j	k0
183	 rfe
184#else
185	and	k0, ST0_IE
186	bnez	k0, 1f
187
188	eret
189#endif
1901:
191	.set pop
192#endif
193	SAVE_ALL docfi=1
194	CLI
195	TRACE_IRQS_OFF
196
197	LONG_L	s0, TI_REGS($28)
198	LONG_S	sp, TI_REGS($28)
199
200	/*
201	 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
202	 * Check if we are already using the IRQ stack.
203	 */
204	move	s1, sp # Preserve the sp
205
206	/* Get IRQ stack for this CPU */
207	ASM_CPUID_MFC0	k0, ASM_SMP_CPUID_REG
208#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
209	lui	k1, %hi(irq_stack)
210#else
211	lui	k1, %highest(irq_stack)
212	daddiu	k1, %higher(irq_stack)
213	dsll	k1, 16
214	daddiu	k1, %hi(irq_stack)
215	dsll	k1, 16
216#endif
217	LONG_SRL	k0, SMP_CPUID_PTRSHIFT
218	LONG_ADDU	k1, k0
219	LONG_L	t0, %lo(irq_stack)(k1)
220
221	# Check if already on IRQ stack
222	PTR_LI	t1, ~(_THREAD_SIZE-1)
223	and	t1, t1, sp
224	beq	t0, t1, 2f
225
226	/* Switch to IRQ stack */
227	li	t1, _IRQ_STACK_START
228	PTR_ADD sp, t0, t1
229
230	/* Save task's sp on IRQ stack so that unwinding can follow it */
231	LONG_S	s1, 0(sp)
2322:
233	jal	plat_irq_dispatch
234
235	/* Restore sp */
236	move	sp, s1
237
238	j	ret_from_irq
239#ifdef CONFIG_CPU_MICROMIPS
240	nop
241#endif
242	END(handle_int)
243
244	__INIT
245
246/*
247 * Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
248 * This is a dedicated interrupt exception vector which reduces the
249 * interrupt processing overhead.  The jump instruction will be replaced
250 * at the initialization time.
251 *
252 * Be careful when changing this, it has to be at most 128 bytes
253 * to fit into space reserved for the exception handler.
254 */
255NESTED(except_vec4, 0, sp)
2561:	j	1b			/* Dummy, will be replaced */
257	END(except_vec4)
258
259/*
260 * EJTAG debug exception handler.
261 * The EJTAG debug exception entry point is 0xbfc00480, which
262 * normally is in the boot PROM, so the boot PROM must do an
263 * unconditional jump to this vector.
264 */
265NESTED(except_vec_ejtag_debug, 0, sp)
266	j	ejtag_debug_handler
267#ifdef CONFIG_CPU_MICROMIPS
268	 nop
269#endif
270	END(except_vec_ejtag_debug)
271
272	__FINIT
273
274/*
275 * Vectored interrupt handler.
276 * This prototype is copied to ebase + n*IntCtl.VS and patched
277 * to invoke the handler
278 */
279BUILD_SKIPOVER_PROLOGUE except_vec_vi
280NESTED(except_vec_vi, 0, sp)
281	SAVE_SOME docfi=1
282	SAVE_AT docfi=1
283	.set	push
284	.set	noreorder
285	PTR_LA	v1, except_vec_vi_handler
286	jr	v1
287FEXPORT(except_vec_vi_ori)
288	 ori	v0, zero, 0		/* Offset in vi_handlers[] */
289	.set	pop
290	END(except_vec_vi)
291EXPORT(except_vec_vi_end)
292
293/*
294 * Common Vectored Interrupt code
295 * Complete the register saves and invoke the handler, $v0 holds
296 * offset into vi_handlers[]
297 */
298NESTED(except_vec_vi_handler, 0, sp)
299	SAVE_TEMP
300	SAVE_STATIC
301	CLI
302#ifdef CONFIG_TRACE_IRQFLAGS
303	move	s0, v0
304	TRACE_IRQS_OFF
305	move	v0, s0
306#endif
307
308	LONG_L	s0, TI_REGS($28)
309	LONG_S	sp, TI_REGS($28)
310
311	/*
312	 * SAVE_ALL ensures we are using a valid kernel stack for the thread.
313	 * Check if we are already using the IRQ stack.
314	 */
315	move	s1, sp # Preserve the sp
316
317	/* Get IRQ stack for this CPU */
318	ASM_CPUID_MFC0	k0, ASM_SMP_CPUID_REG
319#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
320	lui	k1, %hi(irq_stack)
321#else
322	lui	k1, %highest(irq_stack)
323	daddiu	k1, %higher(irq_stack)
324	dsll	k1, 16
325	daddiu	k1, %hi(irq_stack)
326	dsll	k1, 16
327#endif
328	LONG_SRL	k0, SMP_CPUID_PTRSHIFT
329	LONG_ADDU	k1, k0
330	LONG_L	t0, %lo(irq_stack)(k1)
331
332	# Check if already on IRQ stack
333	PTR_LI	t1, ~(_THREAD_SIZE-1)
334	and	t1, t1, sp
335	beq	t0, t1, 2f
336
337	/* Switch to IRQ stack */
338	li	t1, _IRQ_STACK_START
339	PTR_ADD sp, t0, t1
340
341	/* Save task's sp on IRQ stack so that unwinding can follow it */
342	LONG_S	s1, 0(sp)
3432:
344	PTR_L	v0, vi_handlers(v0)
345	jalr	v0
346
347	/* Restore sp */
348	move	sp, s1
349
350	j	ret_from_irq
351	END(except_vec_vi_handler)
352
353/*
354 * EJTAG debug exception handler.
355 */
356NESTED(ejtag_debug_handler, PT_SIZE, sp)
357	.set	push
358	.set	noat
359	MTC0	k0, CP0_DESAVE
360	mfc0	k0, CP0_DEBUG
361
362	andi	k0, k0, MIPS_DEBUG_DBP	# Check for SDBBP.
363	beqz	k0, ejtag_return
364
365#ifdef CONFIG_SMP
3661:	PTR_LA	k0, ejtag_debug_buffer_spinlock
367	__SYNC(full, loongson3_war)
3682:	ll	k0, 0(k0)
369	bnez	k0, 2b
370	PTR_LA	k0, ejtag_debug_buffer_spinlock
371	sc	k0, 0(k0)
372	beqz	k0, 1b
373# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC
374	sync
375# endif
376
377	PTR_LA	k0, ejtag_debug_buffer
378	LONG_S	k1, 0(k0)
379
380	ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
381	PTR_SRL	k1, SMP_CPUID_PTRSHIFT
382	PTR_SLL	k1, LONGLOG
383	PTR_LA	k0, ejtag_debug_buffer_per_cpu
384	PTR_ADDU k0, k1
385
386	PTR_LA	k1, ejtag_debug_buffer
387	LONG_L	k1, 0(k1)
388	LONG_S	k1, 0(k0)
389
390	PTR_LA	k0, ejtag_debug_buffer_spinlock
391	sw	zero, 0(k0)
392#else
393	PTR_LA	k0, ejtag_debug_buffer
394	LONG_S	k1, 0(k0)
395#endif
396
397	SAVE_ALL
398	move	a0, sp
399	jal	ejtag_exception_handler
400	RESTORE_ALL
401
402#ifdef CONFIG_SMP
403	ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
404	PTR_SRL	k1, SMP_CPUID_PTRSHIFT
405	PTR_SLL	k1, LONGLOG
406	PTR_LA	k0, ejtag_debug_buffer_per_cpu
407	PTR_ADDU k0, k1
408	LONG_L	k1, 0(k0)
409#else
410	PTR_LA	k0, ejtag_debug_buffer
411	LONG_L	k1, 0(k0)
412#endif
413
414ejtag_return:
415	back_to_back_c0_hazard
416	MFC0	k0, CP0_DESAVE
417	.set	mips32
418	deret
419	.set	pop
420	END(ejtag_debug_handler)
421
422/*
423 * This buffer is reserved for the use of the EJTAG debug
424 * handler.
425 */
426	.data
427EXPORT(ejtag_debug_buffer)
428	.fill	LONGSIZE
429#ifdef CONFIG_SMP
430EXPORT(ejtag_debug_buffer_spinlock)
431	.fill	LONGSIZE
432EXPORT(ejtag_debug_buffer_per_cpu)
433	.fill	LONGSIZE * NR_CPUS
434#endif
435	.previous
436
437	__INIT
438
439/*
440 * NMI debug exception handler for MIPS reference boards.
441 * The NMI debug exception entry point is 0xbfc00000, which
442 * normally is in the boot PROM, so the boot PROM must do a
443 * unconditional jump to this vector.
444 */
445NESTED(except_vec_nmi, 0, sp)
446	j	nmi_handler
447#ifdef CONFIG_CPU_MICROMIPS
448	 nop
449#endif
450	END(except_vec_nmi)
451
452	__FINIT
453
454NESTED(nmi_handler, PT_SIZE, sp)
455	.cfi_signal_frame
456	.set	push
457	.set	noat
458	/*
459	 * Clear ERL - restore segment mapping
460	 * Clear BEV - required for page fault exception handler to work
461	 */
462	mfc0	k0, CP0_STATUS
463	ori	k0, k0, ST0_EXL
464	li	k1, ~(ST0_BEV | ST0_ERL)
465	and	k0, k0, k1
466	mtc0	k0, CP0_STATUS
467	_ehb
468	SAVE_ALL
469	move	a0, sp
470	jal	nmi_exception_handler
471	/* nmi_exception_handler never returns */
472	.set	pop
473	END(nmi_handler)
474
475	.macro	__build_clear_none
476	.endm
477
478	.macro	__build_clear_sti
479	TRACE_IRQS_ON
480	STI
481	.endm
482
483	.macro	__build_clear_cli
484	CLI
485	TRACE_IRQS_OFF
486	.endm
487
488	.macro	__build_clear_fpe
489	CLI
490	TRACE_IRQS_OFF
491	.set	push
492	/* gas fails to assemble cfc1 for some archs (octeon).*/ \
493	.set	mips1
494	.set	hardfloat
495	cfc1	a1, fcr31
496	.set	pop
497	.endm
498
499	.macro	__build_clear_msa_fpe
500	CLI
501	TRACE_IRQS_OFF
502	_cfcmsa	a1, MSA_CSR
503	.endm
504
505	.macro	__build_clear_ade
506	MFC0	t0, CP0_BADVADDR
507	PTR_S	t0, PT_BVADDR(sp)
508	KMODE
509	.endm
510
511	.macro __build_clear_gsexc
512	.set	push
513	/*
514	 * We need to specify a selector to access the CP0.Diag1 (GSCause)
515	 * register. All GSExc-equipped processors have MIPS32.
516	 */
517	.set	mips32
518	mfc0	a1, CP0_DIAGNOSTIC1
519	.set	pop
520	TRACE_IRQS_ON
521	STI
522	.endm
523
524	.macro	__BUILD_silent exception
525	.endm
526
527	/* Gas tries to parse the ASM_PRINT argument as a string containing
528	   string escapes and emits bogus warnings if it believes to
529	   recognize an unknown escape code.  So make the arguments
530	   start with an n and gas will believe \n is ok ...  */
531	.macro	__BUILD_verbose nexception
532	LONG_L	a1, PT_EPC(sp)
533#ifdef CONFIG_32BIT
534	ASM_PRINT("Got \nexception at %08lx\012")
535#endif
536#ifdef CONFIG_64BIT
537	ASM_PRINT("Got \nexception at %016lx\012")
538#endif
539	.endm
540
541	.macro	__BUILD_count exception
542	LONG_L	t0,exception_count_\exception
543	LONG_ADDIU	t0, 1
544	LONG_S	t0,exception_count_\exception
545	.comm	exception_count\exception, 8, 8
546	.endm
547
548	.macro	__BUILD_HANDLER exception handler clear verbose ext
549	.align	5
550	NESTED(handle_\exception, PT_SIZE, sp)
551	.cfi_signal_frame
552	.set	noat
553	SAVE_ALL
554	FEXPORT(handle_\exception\ext)
555	__build_clear_\clear
556	.set	at
557	__BUILD_\verbose \exception
558	move	a0, sp
559	jal	do_\handler
560	j	ret_from_exception
561	END(handle_\exception)
562	.endm
563
564	.macro	BUILD_HANDLER exception handler clear verbose
565	__BUILD_HANDLER \exception \handler \clear \verbose _int
566	.endm
567
568	BUILD_HANDLER adel ade ade silent		/* #4  */
569	BUILD_HANDLER ades ade ade silent		/* #5  */
570	BUILD_HANDLER ibe be cli silent			/* #6  */
571	BUILD_HANDLER dbe be cli silent			/* #7  */
572	BUILD_HANDLER bp bp sti silent			/* #9  */
573	BUILD_HANDLER ri ri sti silent			/* #10 */
574	BUILD_HANDLER cpu cpu sti silent		/* #11 */
575	BUILD_HANDLER ov ov sti silent			/* #12 */
576	BUILD_HANDLER tr tr sti silent			/* #13 */
577	BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent	/* #14 */
578#ifdef CONFIG_MIPS_FP_SUPPORT
579	BUILD_HANDLER fpe fpe fpe silent		/* #15 */
580#endif
581	BUILD_HANDLER ftlb ftlb none silent		/* #16 */
582	BUILD_HANDLER gsexc gsexc gsexc silent		/* #16 */
583	BUILD_HANDLER msa msa sti silent		/* #21 */
584	BUILD_HANDLER mdmx mdmx sti silent		/* #22 */
585#ifdef	CONFIG_HARDWARE_WATCHPOINTS
586	/*
587	 * For watch, interrupts will be enabled after the watch
588	 * registers are read.
589	 */
590	BUILD_HANDLER watch watch cli silent		/* #23 */
591#else
592	BUILD_HANDLER watch watch sti verbose		/* #23 */
593#endif
594	BUILD_HANDLER mcheck mcheck cli verbose		/* #24 */
595	BUILD_HANDLER mt mt sti silent			/* #25 */
596	BUILD_HANDLER dsp dsp sti silent		/* #26 */
597	BUILD_HANDLER reserved reserved sti verbose	/* others */
598
599	.align	5
600	LEAF(handle_ri_rdhwr_tlbp)
601	.set	push
602	.set	noat
603	.set	noreorder
604	/* check if TLB contains a entry for EPC */
605	MFC0	k1, CP0_ENTRYHI
606	andi	k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX
607	MFC0	k0, CP0_EPC
608	PTR_SRL	k0, _PAGE_SHIFT + 1
609	PTR_SLL	k0, _PAGE_SHIFT + 1
610	or	k1, k0
611	MTC0	k1, CP0_ENTRYHI
612	mtc0_tlbw_hazard
613	tlbp
614	tlb_probe_hazard
615	mfc0	k1, CP0_INDEX
616	.set	pop
617	bltz	k1, handle_ri	/* slow path */
618	/* fall thru */
619	END(handle_ri_rdhwr_tlbp)
620
621	LEAF(handle_ri_rdhwr)
622	.set	push
623	.set	noat
624	.set	noreorder
625	/* MIPS32:    0x7c03e83b: rdhwr v1,$29 */
626	/* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
627	MFC0	k1, CP0_EPC
628#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2)
629	and	k0, k1, 1
630	beqz	k0, 1f
631	 xor	k1, k0
632	lhu	k0, (k1)
633	lhu	k1, 2(k1)
634	ins	k1, k0, 16, 16
635	lui	k0, 0x007d
636	b	docheck
637	 ori	k0, 0x6b3c
6381:
639	lui	k0, 0x7c03
640	lw	k1, (k1)
641	ori	k0, 0xe83b
642#else
643	andi	k0, k1, 1
644	bnez	k0, handle_ri
645	 lui	k0, 0x7c03
646	lw	k1, (k1)
647	ori	k0, 0xe83b
648#endif
649	.set	reorder
650docheck:
651	bne	k0, k1, handle_ri	/* if not ours */
652
653isrdhwr:
654	/* The insn is rdhwr.  No need to check CAUSE.BD here. */
655	get_saved_sp	/* k1 := current_thread_info */
656	.set	noreorder
657	MFC0	k0, CP0_EPC
658#if defined(CONFIG_CPU_R3000)
659	ori	k1, _THREAD_MASK
660	xori	k1, _THREAD_MASK
661	LONG_L	v1, TI_TP_VALUE(k1)
662	LONG_ADDIU	k0, 4
663	jr	k0
664	 rfe
665#else
666#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
667	LONG_ADDIU	k0, 4		/* stall on $k0 */
668#else
669	.set	at=v1
670	LONG_ADDIU	k0, 4
671	.set	noat
672#endif
673	MTC0	k0, CP0_EPC
674	/* I hope three instructions between MTC0 and ERET are enough... */
675	ori	k1, _THREAD_MASK
676	xori	k1, _THREAD_MASK
677	LONG_L	v1, TI_TP_VALUE(k1)
678	.set	push
679	.set	arch=r4000
680	eret
681	.set	pop
682#endif
683	.set	pop
684	END(handle_ri_rdhwr)
685
686#ifdef CONFIG_CPU_R4X00_BUGS64
687/* A temporary overflow handler used by check_daddi(). */
688
689	__INIT
690
691	BUILD_HANDLER  daddi_ov daddi_ov none silent	/* #12 */
692#endif
693