1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 #ifndef _ASM_POWERPC_INTERRUPT_H
3 #define _ASM_POWERPC_INTERRUPT_H
4
5 /* BookE/4xx */
6 #define INTERRUPT_CRITICAL_INPUT 0x100
7
8 /* BookE */
9 #define INTERRUPT_DEBUG 0xd00
10 #ifdef CONFIG_BOOKE
11 #define INTERRUPT_PERFMON 0x260
12 #define INTERRUPT_DOORBELL 0x280
13 #endif
14
15 /* BookS/4xx/8xx */
16 #define INTERRUPT_MACHINE_CHECK 0x200
17
18 /* BookS/8xx */
19 #define INTERRUPT_SYSTEM_RESET 0x100
20
21 /* BookS */
22 #define INTERRUPT_DATA_SEGMENT 0x380
23 #define INTERRUPT_INST_SEGMENT 0x480
24 #define INTERRUPT_TRACE 0xd00
25 #define INTERRUPT_H_DATA_STORAGE 0xe00
26 #define INTERRUPT_HMI 0xe60
27 #define INTERRUPT_H_FAC_UNAVAIL 0xf80
28 #ifdef CONFIG_PPC_BOOK3S
29 #define INTERRUPT_DOORBELL 0xa00
30 #define INTERRUPT_PERFMON 0xf00
31 #define INTERRUPT_ALTIVEC_UNAVAIL 0xf20
32 #endif
33
34 /* BookE/BookS/4xx/8xx */
35 #define INTERRUPT_DATA_STORAGE 0x300
36 #define INTERRUPT_INST_STORAGE 0x400
37 #define INTERRUPT_EXTERNAL 0x500
38 #define INTERRUPT_ALIGNMENT 0x600
39 #define INTERRUPT_PROGRAM 0x700
40 #define INTERRUPT_SYSCALL 0xc00
41 #define INTERRUPT_TRACE 0xd00
42
43 /* BookE/BookS/44x */
44 #define INTERRUPT_FP_UNAVAIL 0x800
45
46 /* BookE/BookS/44x/8xx */
47 #define INTERRUPT_DECREMENTER 0x900
48
49 #ifndef INTERRUPT_PERFMON
50 #define INTERRUPT_PERFMON 0x0
51 #endif
52
53 /* 8xx */
54 #define INTERRUPT_SOFT_EMU_8xx 0x1000
55 #define INTERRUPT_INST_TLB_MISS_8xx 0x1100
56 #define INTERRUPT_DATA_TLB_MISS_8xx 0x1200
57 #define INTERRUPT_INST_TLB_ERROR_8xx 0x1300
58 #define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400
59 #define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00
60 #define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00
61
62 /* 603 */
63 #define INTERRUPT_INST_TLB_MISS_603 0x1000
64 #define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100
65 #define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200
66
67 #ifndef __ASSEMBLER__
68
69 #include <linux/sched/debug.h> /* for show_regs */
70 #include <linux/irq-entry-common.h>
71
72 #include <asm/kprobes.h>
73 #include <asm/runlatch.h>
74
75 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
76 /*
77 * WARN/BUG is handled with a program interrupt so minimise checks here to
78 * avoid recursion and maximise the chance of getting the first oops handled.
79 */
80 #define INT_SOFT_MASK_BUG_ON(regs, cond) \
81 do { \
82 if ((user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM))) \
83 BUG_ON(cond); \
84 } while (0)
85 #else
86 #define INT_SOFT_MASK_BUG_ON(regs, cond)
87 #endif
88
89 /*
90 * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each
91 * function definition. The reason for this is the noinstr section is placed
92 * after the main text section, i.e., very far away from the interrupt entry
93 * asm. That creates problems with fitting linker stubs when building large
94 * kernels.
95 */
96 #define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address
97
98 /**
99 * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function
100 * @func: Function name of the entry point
101 * @returns: Returns a value back to asm caller
102 */
103 #define DECLARE_INTERRUPT_HANDLER_RAW(func) \
104 __visible long func(struct pt_regs *regs)
105
106 /**
107 * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function
108 * @func: Function name of the entry point
109 * @returns: Returns a value back to asm caller
110 *
111 * @func is called from ASM entry code.
112 *
113 * This is a plain function which does no tracing, reconciling, etc.
114 * The macro is written so it acts as function definition. Append the
115 * body with a pair of curly brackets.
116 *
117 * raw interrupt handlers must not enable or disable interrupts, or
118 * schedule, tracing and instrumentation (ftrace, lockdep, etc) would
119 * not be advisable either, although may be possible in a pinch, the
120 * trace will look odd at least.
121 *
122 * A raw handler may call one of the other interrupt handler functions
123 * to be converted into that interrupt context without these restrictions.
124 *
125 * On PPC64, _RAW handlers may return with fast_interrupt_return.
126 *
127 * Specific handlers may have additional restrictions.
128 */
129 #define DEFINE_INTERRUPT_HANDLER_RAW(func) \
130 static __always_inline __no_sanitize_address __no_kcsan long \
131 ____##func(struct pt_regs *regs); \
132 \
133 interrupt_handler long func(struct pt_regs *regs) \
134 { \
135 long ret; \
136 \
137 __hard_RI_enable(); \
138 \
139 ret = ____##func (regs); \
140 \
141 return ret; \
142 } \
143 NOKPROBE_SYMBOL(func); \
144 \
145 static __always_inline __no_sanitize_address __no_kcsan long \
146 ____##func(struct pt_regs *regs)
147
148 /**
149 * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function
150 * @func: Function name of the entry point
151 */
152 #define DECLARE_INTERRUPT_HANDLER(func) \
153 __visible void func(struct pt_regs *regs)
154
155 /**
156 * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function
157 * @func: Function name of the entry point
158 *
159 * @func is called from ASM entry code.
160 *
161 * The macro is written so it acts as function definition. Append the
162 * body with a pair of curly brackets.
163 */
164 #define DEFINE_INTERRUPT_HANDLER(func) \
165 static __always_inline void ____##func(struct pt_regs *regs); \
166 \
167 interrupt_handler void func(struct pt_regs *regs) \
168 { \
169 irqentry_state_t state; \
170 arch_interrupt_enter_prepare(regs); \
171 state = irqentry_enter(regs); \
172 instrumentation_begin(); \
173 ____##func (regs); \
174 instrumentation_end(); \
175 arch_interrupt_exit_prepare(regs); \
176 irqentry_exit(regs, state); \
177 } \
178 NOKPROBE_SYMBOL(func); \
179 \
180 static __always_inline void ____##func(struct pt_regs *regs)
181
182 /**
183 * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function
184 * @func: Function name of the entry point
185 * @returns: Returns a value back to asm caller
186 */
187 #define DECLARE_INTERRUPT_HANDLER_RET(func) \
188 __visible long func(struct pt_regs *regs)
189
190 /**
191 * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function
192 * @func: Function name of the entry point
193 * @returns: Returns a value back to asm caller
194 *
195 * @func is called from ASM entry code.
196 *
197 * The macro is written so it acts as function definition. Append the
198 * body with a pair of curly brackets.
199 */
200 #define DEFINE_INTERRUPT_HANDLER_RET(func) \
201 static __always_inline long ____##func(struct pt_regs *regs); \
202 \
203 interrupt_handler long func(struct pt_regs *regs) \
204 { \
205 long ret; \
206 irqentry_state_t state; \
207 \
208 arch_interrupt_enter_prepare(regs); \
209 state = irqentry_enter(regs); \
210 instrumentation_begin(); \
211 ret = ____##func (regs); \
212 instrumentation_end(); \
213 arch_interrupt_exit_prepare(regs); \
214 irqentry_exit(regs, state); \
215 \
216 return ret; \
217 } \
218 NOKPROBE_SYMBOL(func); \
219 \
220 static __always_inline long ____##func(struct pt_regs *regs)
221
222 /**
223 * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function
224 * @func: Function name of the entry point
225 */
226 #define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \
227 __visible void func(struct pt_regs *regs)
228
229 /**
230 * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function
231 * @func: Function name of the entry point
232 *
233 * @func is called from ASM entry code.
234 *
235 * The macro is written so it acts as function definition. Append the
236 * body with a pair of curly brackets.
237 */
238 #define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \
239 static __always_inline void ____##func(struct pt_regs *regs); \
240 \
241 interrupt_handler void func(struct pt_regs *regs) \
242 { \
243 irqentry_state_t state; \
244 arch_interrupt_async_enter_prepare(regs); \
245 state = irqentry_enter(regs); \
246 instrumentation_begin(); \
247 irq_enter_rcu(); \
248 ____##func (regs); \
249 nap_adjust_return(regs); \
250 irq_exit_rcu(); \
251 instrumentation_end(); \
252 arch_interrupt_async_exit_prepare(regs); \
253 irqentry_exit(regs, state); \
254 } \
255 NOKPROBE_SYMBOL(func); \
256 \
257 static __always_inline void ____##func(struct pt_regs *regs)
258
259 /**
260 * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function
261 * @func: Function name of the entry point
262 * @returns: Returns a value back to asm caller
263 */
264 #define DECLARE_INTERRUPT_HANDLER_NMI(func) \
265 __visible long func(struct pt_regs *regs)
266
267 /**
268 * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function
269 * @func: Function name of the entry point
270 * @returns: Returns a value back to asm caller
271 *
272 * @func is called from ASM entry code.
273 *
274 * The macro is written so it acts as function definition. Append the
275 * body with a pair of curly brackets.
276 */
277 #define DEFINE_INTERRUPT_HANDLER_NMI(func) \
278 static __always_inline __no_sanitize_address __no_kcsan long \
279 ____##func(struct pt_regs *regs); \
280 \
281 interrupt_handler long func(struct pt_regs *regs) \
282 { \
283 irqentry_state_t state; \
284 struct interrupt_nmi_state nmi_state; \
285 long ret; \
286 \
287 arch_interrupt_nmi_enter_prepare(regs, &nmi_state); \
288 if (mfmsr() & MSR_DR) { \
289 /* nmi_entry if relocations are on */ \
290 state = irqentry_nmi_enter(regs); \
291 } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && \
292 firmware_has_feature(FW_FEATURE_LPAR) && \
293 !radix_enabled()) { \
294 /* no nmi_entry for a pseries hash guest \
295 * taking a real mode exception */ \
296 } else if (IS_ENABLED(CONFIG_KASAN)) { \
297 /* no nmi_entry for KASAN in real mode */ \
298 } else if (percpu_first_chunk_is_paged) { \
299 /* no nmi_entry if percpu first chunk is not embedded */\
300 } else { \
301 state = irqentry_nmi_enter(regs); \
302 } \
303 ret = ____##func (regs); \
304 arch_interrupt_nmi_exit_prepare(regs, &nmi_state); \
305 if (mfmsr() & MSR_DR) { \
306 /* nmi_exit if relocations are on */ \
307 irqentry_nmi_exit(regs, state); \
308 } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && \
309 firmware_has_feature(FW_FEATURE_LPAR) && \
310 !radix_enabled()) { \
311 /* no nmi_exit for a pseries hash guest \
312 * taking a real mode exception */ \
313 } else if (IS_ENABLED(CONFIG_KASAN)) { \
314 /* no nmi_exit for KASAN in real mode */ \
315 } else if (percpu_first_chunk_is_paged) { \
316 /* no nmi_exit if percpu first chunk is not embedded */ \
317 } else { \
318 irqentry_nmi_exit(regs, state); \
319 } \
320 \
321 return ret; \
322 } \
323 NOKPROBE_SYMBOL(func); \
324 \
325 static __always_inline __no_sanitize_address __no_kcsan long \
326 ____##func(struct pt_regs *regs)
327
328
329 /* Interrupt handlers */
330 /* kernel/traps.c */
331 DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception);
332 #ifdef CONFIG_PPC_BOOK3S_64
333 DECLARE_INTERRUPT_HANDLER_RAW(machine_check_early_boot);
334 DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async);
335 #endif
336 DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception);
337 DECLARE_INTERRUPT_HANDLER(SMIException);
338 DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
339 DECLARE_INTERRUPT_HANDLER(unknown_exception);
340 DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
341 DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception);
342 DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
343 DECLARE_INTERRUPT_HANDLER(RunModeException);
344 DECLARE_INTERRUPT_HANDLER(single_step_exception);
345 DECLARE_INTERRUPT_HANDLER(program_check_exception);
346 DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt);
347 DECLARE_INTERRUPT_HANDLER(alignment_exception);
348 DECLARE_INTERRUPT_HANDLER(StackOverflow);
349 DECLARE_INTERRUPT_HANDLER(stack_overflow_exception);
350 DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception);
351 DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception);
352 DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception);
353 DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception);
354 DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm);
355 DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm);
356 DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm);
357 DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi);
358 DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async);
359 DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception);
360 DECLARE_INTERRUPT_HANDLER(DebugException);
361 DECLARE_INTERRUPT_HANDLER(altivec_assist_exception);
362 DECLARE_INTERRUPT_HANDLER(CacheLockingException);
363 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);
364 DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);
365 DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException);
366 DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
367
368 /* slb.c */
369 DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault);
370 DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt);
371
372 /* hash_utils.c */
373 DECLARE_INTERRUPT_HANDLER(do_hash_fault);
374
375 /* fault.c */
376 DECLARE_INTERRUPT_HANDLER(do_page_fault);
377 DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);
378
379 /* process.c */
380 DECLARE_INTERRUPT_HANDLER(do_break);
381
382 /* time.c */
383 DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt);
384
385 /* mce.c */
386 DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early);
387 DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode);
388
389 DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);
390
391 /* irq.c */
392 DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ);
393
394 void __noreturn unrecoverable_exception(struct pt_regs *regs);
395
396 void replay_system_reset(void);
397 void replay_soft_interrupts(void);
398
interrupt_cond_local_irq_enable(struct pt_regs * regs)399 static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs)
400 {
401 if (!regs_irqs_disabled(regs))
402 local_irq_enable();
403 }
404
405 long system_call_exception(struct pt_regs *regs, unsigned long r0);
406 notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
407 notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs);
408 notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs);
409 #ifdef CONFIG_PPC64
410 unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs);
411 unsigned long interrupt_exit_user_restart(struct pt_regs *regs);
412 unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs);
413 #endif
414
415 #endif /* __ASSEMBLER__ */
416
417 #endif /* _ASM_POWERPC_INTERRUPT_H */
418