18d41fc61SNicholas Piggin /* SPDX-License-Identifier: GPL-2.0-or-later */
28d41fc61SNicholas Piggin #ifndef _ASM_POWERPC_INTERRUPT_H
38d41fc61SNicholas Piggin #define _ASM_POWERPC_INTERRUPT_H
48d41fc61SNicholas Piggin
57153d4bfSXiongwei Song /* BookE/4xx */
67153d4bfSXiongwei Song #define INTERRUPT_CRITICAL_INPUT 0x100
77153d4bfSXiongwei Song
87153d4bfSXiongwei Song /* BookE */
97153d4bfSXiongwei Song #define INTERRUPT_DEBUG 0xd00
107153d4bfSXiongwei Song #ifdef CONFIG_BOOKE
117153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x260
127153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0x280
137153d4bfSXiongwei Song #endif
147153d4bfSXiongwei Song
157153d4bfSXiongwei Song /* BookS/4xx/8xx */
167153d4bfSXiongwei Song #define INTERRUPT_MACHINE_CHECK 0x200
177153d4bfSXiongwei Song
187153d4bfSXiongwei Song /* BookS/8xx */
197153d4bfSXiongwei Song #define INTERRUPT_SYSTEM_RESET 0x100
207153d4bfSXiongwei Song
217153d4bfSXiongwei Song /* BookS */
227153d4bfSXiongwei Song #define INTERRUPT_DATA_SEGMENT 0x380
237153d4bfSXiongwei Song #define INTERRUPT_INST_SEGMENT 0x480
247153d4bfSXiongwei Song #define INTERRUPT_TRACE 0xd00
257153d4bfSXiongwei Song #define INTERRUPT_H_DATA_STORAGE 0xe00
26e5223311SChristophe Leroy #define INTERRUPT_HMI 0xe60
277153d4bfSXiongwei Song #define INTERRUPT_H_FAC_UNAVAIL 0xf80
287153d4bfSXiongwei Song #ifdef CONFIG_PPC_BOOK3S
297153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0xa00
307153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0xf00
317fab6397SChristophe Leroy #define INTERRUPT_ALTIVEC_UNAVAIL 0xf20
327153d4bfSXiongwei Song #endif
337153d4bfSXiongwei Song
347153d4bfSXiongwei Song /* BookE/BookS/4xx/8xx */
357153d4bfSXiongwei Song #define INTERRUPT_DATA_STORAGE 0x300
367153d4bfSXiongwei Song #define INTERRUPT_INST_STORAGE 0x400
370f5eb28aSChristophe Leroy #define INTERRUPT_EXTERNAL 0x500
387153d4bfSXiongwei Song #define INTERRUPT_ALIGNMENT 0x600
397153d4bfSXiongwei Song #define INTERRUPT_PROGRAM 0x700
407153d4bfSXiongwei Song #define INTERRUPT_SYSCALL 0xc00
410f5eb28aSChristophe Leroy #define INTERRUPT_TRACE 0xd00
427153d4bfSXiongwei Song
437153d4bfSXiongwei Song /* BookE/BookS/44x */
447153d4bfSXiongwei Song #define INTERRUPT_FP_UNAVAIL 0x800
457153d4bfSXiongwei Song
467153d4bfSXiongwei Song /* BookE/BookS/44x/8xx */
477153d4bfSXiongwei Song #define INTERRUPT_DECREMENTER 0x900
487153d4bfSXiongwei Song
497153d4bfSXiongwei Song #ifndef INTERRUPT_PERFMON
507153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x0
517153d4bfSXiongwei Song #endif
527153d4bfSXiongwei Song
530f5eb28aSChristophe Leroy /* 8xx */
540f5eb28aSChristophe Leroy #define INTERRUPT_SOFT_EMU_8xx 0x1000
550f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_MISS_8xx 0x1100
560f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_MISS_8xx 0x1200
570f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_ERROR_8xx 0x1300
580f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400
590f5eb28aSChristophe Leroy #define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00
600f5eb28aSChristophe Leroy #define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00
610f5eb28aSChristophe Leroy
627fab6397SChristophe Leroy /* 603 */
637fab6397SChristophe Leroy #define INTERRUPT_INST_TLB_MISS_603 0x1000
647fab6397SChristophe Leroy #define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100
657fab6397SChristophe Leroy #define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200
667fab6397SChristophe Leroy
670f5eb28aSChristophe Leroy #ifndef __ASSEMBLY__
680f5eb28aSChristophe Leroy
690f5eb28aSChristophe Leroy #include <linux/context_tracking.h>
700f5eb28aSChristophe Leroy #include <linux/hardirq.h>
710f5eb28aSChristophe Leroy #include <asm/cputime.h>
7246d60bdbSChristophe Leroy #include <asm/firmware.h>
730f5eb28aSChristophe Leroy #include <asm/ftrace.h>
740f5eb28aSChristophe Leroy #include <asm/kprobes.h>
750f5eb28aSChristophe Leroy #include <asm/runlatch.h>
760f5eb28aSChristophe Leroy
77fb3b72a3SNicholas Piggin #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
78f7bff6e7SNicholas Piggin /*
79f7bff6e7SNicholas Piggin * WARN/BUG is handled with a program interrupt so minimise checks here to
80f7bff6e7SNicholas Piggin * avoid recursion and maximise the chance of getting the first oops handled.
81f7bff6e7SNicholas Piggin */
82f7bff6e7SNicholas Piggin #define INT_SOFT_MASK_BUG_ON(regs, cond) \
83f7bff6e7SNicholas Piggin do { \
84fb3b72a3SNicholas Piggin if ((user_mode(regs) || (TRAP(regs) != INTERRUPT_PROGRAM))) \
85f7bff6e7SNicholas Piggin BUG_ON(cond); \
86f7bff6e7SNicholas Piggin } while (0)
87fb3b72a3SNicholas Piggin #else
88fb3b72a3SNicholas Piggin #define INT_SOFT_MASK_BUG_ON(regs, cond)
89f7bff6e7SNicholas Piggin #endif
90f7bff6e7SNicholas Piggin
919b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64
92f23699c9SNicholas Piggin extern char __end_soft_masked[];
93325678fdSNicholas Piggin bool search_kernel_soft_mask_table(unsigned long addr);
94f23699c9SNicholas Piggin unsigned long search_kernel_restart_table(unsigned long addr);
95f23699c9SNicholas Piggin
9613799748SNicholas Piggin DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);
9713799748SNicholas Piggin
is_implicit_soft_masked(struct pt_regs * regs)989b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs)
999b69d48cSNicholas Piggin {
100d5835fb6SChristophe Leroy if (user_mode(regs))
1019b69d48cSNicholas Piggin return false;
1029b69d48cSNicholas Piggin
1039b69d48cSNicholas Piggin if (regs->nip >= (unsigned long)__end_soft_masked)
1049b69d48cSNicholas Piggin return false;
1059b69d48cSNicholas Piggin
106325678fdSNicholas Piggin return search_kernel_soft_mask_table(regs->nip);
1079b69d48cSNicholas Piggin }
1089b69d48cSNicholas Piggin
srr_regs_clobbered(void)10959dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void)
11059dc5bfcSNicholas Piggin {
11159dc5bfcSNicholas Piggin local_paca->srr_valid = 0;
11259dc5bfcSNicholas Piggin local_paca->hsrr_valid = 0;
11359dc5bfcSNicholas Piggin }
11459dc5bfcSNicholas Piggin #else
search_kernel_restart_table(unsigned long addr)115ecb1057cSNicholas Piggin static inline unsigned long search_kernel_restart_table(unsigned long addr)
116ecb1057cSNicholas Piggin {
117ecb1057cSNicholas Piggin return 0;
118ecb1057cSNicholas Piggin }
119ecb1057cSNicholas Piggin
is_implicit_soft_masked(struct pt_regs * regs)1209b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs)
1219b69d48cSNicholas Piggin {
1229b69d48cSNicholas Piggin return false;
1239b69d48cSNicholas Piggin }
1249b69d48cSNicholas Piggin
srr_regs_clobbered(void)12559dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void)
12659dc5bfcSNicholas Piggin {
12759dc5bfcSNicholas Piggin }
12859dc5bfcSNicholas Piggin #endif
12959dc5bfcSNicholas Piggin
nap_adjust_return(struct pt_regs * regs)13098db179aSNicholas Piggin static inline void nap_adjust_return(struct pt_regs *regs)
13198db179aSNicholas Piggin {
13298db179aSNicholas Piggin #ifdef CONFIG_PPC_970_NAP
13398db179aSNicholas Piggin if (unlikely(test_thread_local_flags(_TLF_NAPPING))) {
13498db179aSNicholas Piggin /* Can avoid a test-and-clear because NMIs do not call this */
13598db179aSNicholas Piggin clear_thread_local_flags(_TLF_NAPPING);
13659dc5bfcSNicholas Piggin regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return);
13798db179aSNicholas Piggin }
13898db179aSNicholas Piggin #endif
13998db179aSNicholas Piggin }
14098db179aSNicholas Piggin
booke_restore_dbcr0(void)141d524dda7SChristophe Leroy static inline void booke_restore_dbcr0(void)
142d524dda7SChristophe Leroy {
143d524dda7SChristophe Leroy #ifdef CONFIG_PPC_ADV_DEBUG_REGS
144d524dda7SChristophe Leroy unsigned long dbcr0 = current->thread.debug.dbcr0;
145d524dda7SChristophe Leroy
146d524dda7SChristophe Leroy if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) {
147d524dda7SChristophe Leroy mtspr(SPRN_DBSR, -1);
148d524dda7SChristophe Leroy mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]);
149d524dda7SChristophe Leroy }
150d524dda7SChristophe Leroy #endif
151d524dda7SChristophe Leroy }
152d524dda7SChristophe Leroy
interrupt_enter_prepare(struct pt_regs * regs)153973e2e64SChristophe Leroy static inline void interrupt_enter_prepare(struct pt_regs *regs)
15425b7e6bbSNicholas Piggin {
155097157e1SNicholas Piggin #ifdef CONFIG_PPC64
156ff0b0d6eSNicholas Piggin irq_soft_mask_set(IRQS_ALL_DISABLED);
1574423eb5aSNicholas Piggin
1584423eb5aSNicholas Piggin /*
1594423eb5aSNicholas Piggin * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE].
1604423eb5aSNicholas Piggin * Asynchronous interrupts get here with HARD_DIS set (see below), so
1614423eb5aSNicholas Piggin * this enables MSR[EE] for synchronous interrupts. IRQs remain
1624423eb5aSNicholas Piggin * soft-masked. The interrupt handler may later call
1634423eb5aSNicholas Piggin * interrupt_cond_local_irq_enable() to achieve a regular process
1644423eb5aSNicholas Piggin * context.
1654423eb5aSNicholas Piggin */
1664423eb5aSNicholas Piggin if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) {
167f7bff6e7SNicholas Piggin INT_SOFT_MASK_BUG_ON(regs, !(regs->msr & MSR_EE));
1684423eb5aSNicholas Piggin __hard_irq_enable();
169ff0b0d6eSNicholas Piggin } else {
170ff0b0d6eSNicholas Piggin __hard_RI_enable();
1714423eb5aSNicholas Piggin }
172fb3b72a3SNicholas Piggin /* Enable MSR[RI] early, to support kernel SLB and hash faults */
173fb3b72a3SNicholas Piggin #endif
17475b96950SNicholas Piggin
175fb3b72a3SNicholas Piggin if (!arch_irq_disabled_regs(regs))
176ff0b0d6eSNicholas Piggin trace_hardirqs_off();
177ff0b0d6eSNicholas Piggin
178f821bc97SNicholas Piggin if (user_mode(regs)) {
17942e03bc5SChristophe Leroy kuap_lock();
180*d65d411cSValentin Schneider CT_WARN_ON(ct_state() != CT_STATE_USER);
181f821bc97SNicholas Piggin user_exit_irqoff();
18256acfdd8SNicholas Piggin
18356acfdd8SNicholas Piggin account_cpu_user_entry();
18456acfdd8SNicholas Piggin account_stolen_time();
185f821bc97SNicholas Piggin } else {
18642e03bc5SChristophe Leroy kuap_save_and_lock(regs);
187f821bc97SNicholas Piggin /*
188f821bc97SNicholas Piggin * CT_WARN_ON comes here via program_check_exception,
189f821bc97SNicholas Piggin * so avoid recursion.
190f821bc97SNicholas Piggin */
191f7bff6e7SNicholas Piggin if (TRAP(regs) != INTERRUPT_PROGRAM)
192*d65d411cSValentin Schneider CT_WARN_ON(ct_state() != CT_STATE_KERNEL &&
193*d65d411cSValentin Schneider ct_state() != CT_STATE_IDLE);
194f7bff6e7SNicholas Piggin INT_SOFT_MASK_BUG_ON(regs, is_implicit_soft_masked(regs));
195f7bff6e7SNicholas Piggin INT_SOFT_MASK_BUG_ON(regs, arch_irq_disabled_regs(regs) &&
196f7bff6e7SNicholas Piggin search_kernel_restart_table(regs->nip));
1979d1988caSNicholas Piggin }
198f7bff6e7SNicholas Piggin INT_SOFT_MASK_BUG_ON(regs, !arch_irq_disabled_regs(regs) &&
199f7bff6e7SNicholas Piggin !(regs->msr & MSR_EE));
20075b96950SNicholas Piggin
20179f4bb17SChristophe Leroy booke_restore_dbcr0();
20225b7e6bbSNicholas Piggin }
20325b7e6bbSNicholas Piggin
20425b7e6bbSNicholas Piggin /*
20525b7e6bbSNicholas Piggin * Care should be taken to note that interrupt_exit_prepare and
20625b7e6bbSNicholas Piggin * interrupt_async_exit_prepare do not necessarily return immediately to
20725b7e6bbSNicholas Piggin * regs context (e.g., if regs is usermode, we don't necessarily return to
20825b7e6bbSNicholas Piggin * user mode). Other interrupts might be taken between here and return,
20925b7e6bbSNicholas Piggin * context switch / preemption may occur in the exit path after this, or a
21025b7e6bbSNicholas Piggin * signal may be delivered, etc.
21125b7e6bbSNicholas Piggin *
21225b7e6bbSNicholas Piggin * The real interrupt exit code is platform specific, e.g.,
21325b7e6bbSNicholas Piggin * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s.
21425b7e6bbSNicholas Piggin *
21525b7e6bbSNicholas Piggin * However interrupt_nmi_exit_prepare does return directly to regs, because
21625b7e6bbSNicholas Piggin * NMIs do not do "exit work" or replay soft-masked interrupts.
21725b7e6bbSNicholas Piggin */
interrupt_exit_prepare(struct pt_regs * regs)218973e2e64SChristophe Leroy static inline void interrupt_exit_prepare(struct pt_regs *regs)
21925b7e6bbSNicholas Piggin {
22025b7e6bbSNicholas Piggin }
22125b7e6bbSNicholas Piggin
interrupt_async_enter_prepare(struct pt_regs * regs)222973e2e64SChristophe Leroy static inline void interrupt_async_enter_prepare(struct pt_regs *regs)
22325b7e6bbSNicholas Piggin {
2244423eb5aSNicholas Piggin #ifdef CONFIG_PPC64
2254423eb5aSNicholas Piggin /* Ensure interrupt_enter_prepare does not enable MSR[EE] */
2264423eb5aSNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
2274423eb5aSNicholas Piggin #endif
228973e2e64SChristophe Leroy interrupt_enter_prepare(regs);
22986dbb394SNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64
230ff0b0d6eSNicholas Piggin /*
231ff0b0d6eSNicholas Piggin * RI=1 is set by interrupt_enter_prepare, so this thread flags access
232ff0b0d6eSNicholas Piggin * has to come afterward (it can cause SLB faults).
233ff0b0d6eSNicholas Piggin */
23486dbb394SNicholas Piggin if (cpu_has_feature(CPU_FTR_CTRL) &&
23586dbb394SNicholas Piggin !test_thread_local_flags(_TLF_RUNLATCH))
23686dbb394SNicholas Piggin __ppc64_runlatch_on();
23786dbb394SNicholas Piggin #endif
2381b1b6a6fSNicholas Piggin irq_enter();
23925b7e6bbSNicholas Piggin }
24025b7e6bbSNicholas Piggin
interrupt_async_exit_prepare(struct pt_regs * regs)241973e2e64SChristophe Leroy static inline void interrupt_async_exit_prepare(struct pt_regs *regs)
24225b7e6bbSNicholas Piggin {
24398db179aSNicholas Piggin /*
24498db179aSNicholas Piggin * Adjust at exit so the main handler sees the true NIA. This must
24598db179aSNicholas Piggin * come before irq_exit() because irq_exit can enable interrupts, and
24698db179aSNicholas Piggin * if another interrupt is taken before nap_adjust_return has run
24798db179aSNicholas Piggin * here, then that interrupt would return directly to idle nap return.
24898db179aSNicholas Piggin */
24998db179aSNicholas Piggin nap_adjust_return(regs);
25098db179aSNicholas Piggin
2511b1b6a6fSNicholas Piggin irq_exit();
252973e2e64SChristophe Leroy interrupt_exit_prepare(regs);
25325b7e6bbSNicholas Piggin }
25425b7e6bbSNicholas Piggin
25525b7e6bbSNicholas Piggin struct interrupt_nmi_state {
256118178e6SNicholas Piggin #ifdef CONFIG_PPC64
2576ecbb582SNicholas Piggin u8 irq_soft_mask;
2586ecbb582SNicholas Piggin u8 irq_happened;
259118178e6SNicholas Piggin u8 ftrace_enabled;
2601b048222SNicholas Piggin u64 softe;
261118178e6SNicholas Piggin #endif
26225b7e6bbSNicholas Piggin };
26325b7e6bbSNicholas Piggin
nmi_disables_ftrace(struct pt_regs * regs)2643db8aa10SNicholas Piggin static inline bool nmi_disables_ftrace(struct pt_regs *regs)
2653db8aa10SNicholas Piggin {
2663db8aa10SNicholas Piggin /* Allow DEC and PMI to be traced when they are soft-NMI */
2673db8aa10SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
2687153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_DECREMENTER)
2693db8aa10SNicholas Piggin return false;
2707153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON)
2713db8aa10SNicholas Piggin return false;
2723db8aa10SNicholas Piggin }
273e0d68273SChristophe Leroy if (IS_ENABLED(CONFIG_PPC_BOOK3E_64)) {
2747153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON)
2753db8aa10SNicholas Piggin return false;
2763db8aa10SNicholas Piggin }
2773db8aa10SNicholas Piggin
2783db8aa10SNicholas Piggin return true;
2793db8aa10SNicholas Piggin }
2803db8aa10SNicholas Piggin
interrupt_nmi_enter_prepare(struct pt_regs * regs,struct interrupt_nmi_state * state)28125b7e6bbSNicholas Piggin static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
28225b7e6bbSNicholas Piggin {
283118178e6SNicholas Piggin #ifdef CONFIG_PPC64
2846ecbb582SNicholas Piggin state->irq_soft_mask = local_paca->irq_soft_mask;
2856ecbb582SNicholas Piggin state->irq_happened = local_paca->irq_happened;
2861b048222SNicholas Piggin state->softe = regs->softe;
2876ecbb582SNicholas Piggin
2886ecbb582SNicholas Piggin /*
2896ecbb582SNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does
2906ecbb582SNicholas Piggin * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile
2916ecbb582SNicholas Piggin * because that goes through irq tracing which we don't want in NMI.
2926ecbb582SNicholas Piggin */
2936ecbb582SNicholas Piggin local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
2946ecbb582SNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
2956ecbb582SNicholas Piggin
296768c4701SNicholas Piggin if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) {
297768c4701SNicholas Piggin /*
298768c4701SNicholas Piggin * Adjust regs->softe to be soft-masked if it had not been
299768c4701SNicholas Piggin * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe
300768c4701SNicholas Piggin * not yet set disabled), or if it was in an implicit soft
301768c4701SNicholas Piggin * masked state. This makes arch_irq_disabled_regs(regs)
302768c4701SNicholas Piggin * behave as expected.
303768c4701SNicholas Piggin */
3044ec5feecSNicholas Piggin regs->softe = IRQS_ALL_DISABLED;
3054ec5feecSNicholas Piggin }
3064ec5feecSNicholas Piggin
307ff0b0d6eSNicholas Piggin __hard_RI_enable();
308ff0b0d6eSNicholas Piggin
3096ecbb582SNicholas Piggin /* Don't do any per-CPU operations until interrupt state is fixed */
3103db8aa10SNicholas Piggin
3113db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs)) {
312118178e6SNicholas Piggin state->ftrace_enabled = this_cpu_get_ftrace_enabled();
313118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(0);
314118178e6SNicholas Piggin }
315118178e6SNicholas Piggin #endif
316118178e6SNicholas Piggin
3175352090aSDaniel Axtens /* If data relocations are enabled, it's safe to use nmi_enter() */
3185352090aSDaniel Axtens if (mfmsr() & MSR_DR) {
3195352090aSDaniel Axtens nmi_enter();
3205352090aSDaniel Axtens return;
3215352090aSDaniel Axtens }
3225352090aSDaniel Axtens
323118178e6SNicholas Piggin /*
3245352090aSDaniel Axtens * But do not use nmi_enter() for pseries hash guest taking a real-mode
325118178e6SNicholas Piggin * NMI because not everything it touches is within the RMA limit.
326118178e6SNicholas Piggin */
3275352090aSDaniel Axtens if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
3285352090aSDaniel Axtens firmware_has_feature(FW_FEATURE_LPAR) &&
3295352090aSDaniel Axtens !radix_enabled())
3305352090aSDaniel Axtens return;
3315352090aSDaniel Axtens
3325352090aSDaniel Axtens /*
3335352090aSDaniel Axtens * Likewise, don't use it if we have some form of instrumentation (like
3345352090aSDaniel Axtens * KASAN shadow) that is not safe to access in real mode (even on radix)
3355352090aSDaniel Axtens */
3365352090aSDaniel Axtens if (IS_ENABLED(CONFIG_KASAN))
3375352090aSDaniel Axtens return;
3385352090aSDaniel Axtens
3390db880fcSMahesh Salgaonkar /*
3400db880fcSMahesh Salgaonkar * Likewise, do not use it in real mode if percpu first chunk is not
3410db880fcSMahesh Salgaonkar * embedded. With CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK enabled there
3420db880fcSMahesh Salgaonkar * are chances where percpu allocation can come from vmalloc area.
3430db880fcSMahesh Salgaonkar */
3440db880fcSMahesh Salgaonkar if (percpu_first_chunk_is_paged)
3450db880fcSMahesh Salgaonkar return;
3460db880fcSMahesh Salgaonkar
3475352090aSDaniel Axtens /* Otherwise, it should be safe to call it */
348118178e6SNicholas Piggin nmi_enter();
34925b7e6bbSNicholas Piggin }
35025b7e6bbSNicholas Piggin
interrupt_nmi_exit_prepare(struct pt_regs * regs,struct interrupt_nmi_state * state)35125b7e6bbSNicholas Piggin static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state)
35225b7e6bbSNicholas Piggin {
3535352090aSDaniel Axtens if (mfmsr() & MSR_DR) {
3545352090aSDaniel Axtens // nmi_exit if relocations are on
355118178e6SNicholas Piggin nmi_exit();
3565352090aSDaniel Axtens } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) &&
3575352090aSDaniel Axtens firmware_has_feature(FW_FEATURE_LPAR) &&
3585352090aSDaniel Axtens !radix_enabled()) {
3595352090aSDaniel Axtens // no nmi_exit for a pseries hash guest taking a real mode exception
3605352090aSDaniel Axtens } else if (IS_ENABLED(CONFIG_KASAN)) {
3615352090aSDaniel Axtens // no nmi_exit for KASAN in real mode
3620db880fcSMahesh Salgaonkar } else if (percpu_first_chunk_is_paged) {
3630db880fcSMahesh Salgaonkar // no nmi_exit if percpu first chunk is not embedded
3645352090aSDaniel Axtens } else {
3655352090aSDaniel Axtens nmi_exit();
3665352090aSDaniel Axtens }
367118178e6SNicholas Piggin
36898db179aSNicholas Piggin /*
36998db179aSNicholas Piggin * nmi does not call nap_adjust_return because nmi should not create
37098db179aSNicholas Piggin * new work to do (must use irq_work for that).
37198db179aSNicholas Piggin */
37298db179aSNicholas Piggin
373118178e6SNicholas Piggin #ifdef CONFIG_PPC64
3749b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S
375f23699c9SNicholas Piggin if (arch_irq_disabled_regs(regs)) {
376f23699c9SNicholas Piggin unsigned long rst = search_kernel_restart_table(regs->nip);
377f23699c9SNicholas Piggin if (rst)
378f23699c9SNicholas Piggin regs_set_return_ip(regs, rst);
379f23699c9SNicholas Piggin }
380f23699c9SNicholas Piggin #endif
381f23699c9SNicholas Piggin
3823db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs))
383118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(state->ftrace_enabled);
3846ecbb582SNicholas Piggin
3856ecbb582SNicholas Piggin /* Check we didn't change the pending interrupt mask. */
3866ecbb582SNicholas Piggin WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened);
3871b048222SNicholas Piggin regs->softe = state->softe;
3886ecbb582SNicholas Piggin local_paca->irq_happened = state->irq_happened;
3896ecbb582SNicholas Piggin local_paca->irq_soft_mask = state->irq_soft_mask;
3906ecbb582SNicholas Piggin #endif
39125b7e6bbSNicholas Piggin }
39225b7e6bbSNicholas Piggin
393e4bb64c7SNicholas Piggin /*
394e4bb64c7SNicholas Piggin * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each
395e4bb64c7SNicholas Piggin * function definition. The reason for this is the noinstr section is placed
396e4bb64c7SNicholas Piggin * after the main text section, i.e., very far away from the interrupt entry
397e4bb64c7SNicholas Piggin * asm. That creates problems with fitting linker stubs when building large
398e4bb64c7SNicholas Piggin * kernels.
399e4bb64c7SNicholas Piggin */
400e4bb64c7SNicholas Piggin #define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address
401e4bb64c7SNicholas Piggin
4028d41fc61SNicholas Piggin /**
4038d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function
4048d41fc61SNicholas Piggin * @func: Function name of the entry point
4058d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
4068d41fc61SNicholas Piggin */
4078d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RAW(func) \
4088d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs)
4098d41fc61SNicholas Piggin
4108d41fc61SNicholas Piggin /**
4118d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function
4128d41fc61SNicholas Piggin * @func: Function name of the entry point
4138d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
4148d41fc61SNicholas Piggin *
4158d41fc61SNicholas Piggin * @func is called from ASM entry code.
4168d41fc61SNicholas Piggin *
4178d41fc61SNicholas Piggin * This is a plain function which does no tracing, reconciling, etc.
4188d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the
4198d41fc61SNicholas Piggin * body with a pair of curly brackets.
4208d41fc61SNicholas Piggin *
4218d41fc61SNicholas Piggin * raw interrupt handlers must not enable or disable interrupts, or
4228d41fc61SNicholas Piggin * schedule, tracing and instrumentation (ftrace, lockdep, etc) would
4238d41fc61SNicholas Piggin * not be advisable either, although may be possible in a pinch, the
4248d41fc61SNicholas Piggin * trace will look odd at least.
4258d41fc61SNicholas Piggin *
4268d41fc61SNicholas Piggin * A raw handler may call one of the other interrupt handler functions
4278d41fc61SNicholas Piggin * to be converted into that interrupt context without these restrictions.
4288d41fc61SNicholas Piggin *
4298d41fc61SNicholas Piggin * On PPC64, _RAW handlers may return with fast_interrupt_return.
4308d41fc61SNicholas Piggin *
4318d41fc61SNicholas Piggin * Specific handlers may have additional restrictions.
4328d41fc61SNicholas Piggin */
4338d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RAW(func) \
4345352090aSDaniel Axtens static __always_inline __no_sanitize_address __no_kcsan long \
4355352090aSDaniel Axtens ____##func(struct pt_regs *regs); \
4368d41fc61SNicholas Piggin \
437e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \
4388d41fc61SNicholas Piggin { \
4398d41fc61SNicholas Piggin long ret; \
4408d41fc61SNicholas Piggin \
441ff0b0d6eSNicholas Piggin __hard_RI_enable(); \
442ff0b0d6eSNicholas Piggin \
4438d41fc61SNicholas Piggin ret = ____##func (regs); \
4448d41fc61SNicholas Piggin \
4458d41fc61SNicholas Piggin return ret; \
4468d41fc61SNicholas Piggin } \
447e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \
4488d41fc61SNicholas Piggin \
4495352090aSDaniel Axtens static __always_inline __no_sanitize_address __no_kcsan long \
4505352090aSDaniel Axtens ____##func(struct pt_regs *regs)
4518d41fc61SNicholas Piggin
4528d41fc61SNicholas Piggin /**
4538d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function
4548d41fc61SNicholas Piggin * @func: Function name of the entry point
4558d41fc61SNicholas Piggin */
4568d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER(func) \
4578d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs)
4588d41fc61SNicholas Piggin
4598d41fc61SNicholas Piggin /**
4608d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function
4618d41fc61SNicholas Piggin * @func: Function name of the entry point
4628d41fc61SNicholas Piggin *
4638d41fc61SNicholas Piggin * @func is called from ASM entry code.
4648d41fc61SNicholas Piggin *
4658d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the
4668d41fc61SNicholas Piggin * body with a pair of curly brackets.
4678d41fc61SNicholas Piggin */
4688d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER(func) \
4698d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \
4708d41fc61SNicholas Piggin \
471e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \
4728d41fc61SNicholas Piggin { \
473973e2e64SChristophe Leroy interrupt_enter_prepare(regs); \
47425b7e6bbSNicholas Piggin \
4758d41fc61SNicholas Piggin ____##func (regs); \
47625b7e6bbSNicholas Piggin \
477973e2e64SChristophe Leroy interrupt_exit_prepare(regs); \
4788d41fc61SNicholas Piggin } \
479e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \
4808d41fc61SNicholas Piggin \
4818d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs)
4828d41fc61SNicholas Piggin
4838d41fc61SNicholas Piggin /**
4848d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function
4858d41fc61SNicholas Piggin * @func: Function name of the entry point
4868d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
4878d41fc61SNicholas Piggin */
4888d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RET(func) \
4898d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs)
4908d41fc61SNicholas Piggin
4918d41fc61SNicholas Piggin /**
4928d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function
4938d41fc61SNicholas Piggin * @func: Function name of the entry point
4948d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
4958d41fc61SNicholas Piggin *
4968d41fc61SNicholas Piggin * @func is called from ASM entry code.
4978d41fc61SNicholas Piggin *
4988d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the
4998d41fc61SNicholas Piggin * body with a pair of curly brackets.
5008d41fc61SNicholas Piggin */
5018d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RET(func) \
5028d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \
5038d41fc61SNicholas Piggin \
504e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \
5058d41fc61SNicholas Piggin { \
5068d41fc61SNicholas Piggin long ret; \
5078d41fc61SNicholas Piggin \
508973e2e64SChristophe Leroy interrupt_enter_prepare(regs); \
50925b7e6bbSNicholas Piggin \
5108d41fc61SNicholas Piggin ret = ____##func (regs); \
5118d41fc61SNicholas Piggin \
512973e2e64SChristophe Leroy interrupt_exit_prepare(regs); \
51325b7e6bbSNicholas Piggin \
5148d41fc61SNicholas Piggin return ret; \
5158d41fc61SNicholas Piggin } \
516e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \
5178d41fc61SNicholas Piggin \
5188d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs)
5198d41fc61SNicholas Piggin
5208d41fc61SNicholas Piggin /**
5218d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function
5228d41fc61SNicholas Piggin * @func: Function name of the entry point
5238d41fc61SNicholas Piggin */
5248d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \
5258d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs)
5268d41fc61SNicholas Piggin
5278d41fc61SNicholas Piggin /**
5288d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function
5298d41fc61SNicholas Piggin * @func: Function name of the entry point
5308d41fc61SNicholas Piggin *
5318d41fc61SNicholas Piggin * @func is called from ASM entry code.
5328d41fc61SNicholas Piggin *
5338d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the
5348d41fc61SNicholas Piggin * body with a pair of curly brackets.
5358d41fc61SNicholas Piggin */
5368d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \
5378d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \
5388d41fc61SNicholas Piggin \
539e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \
5408d41fc61SNicholas Piggin { \
541973e2e64SChristophe Leroy interrupt_async_enter_prepare(regs); \
54225b7e6bbSNicholas Piggin \
5438d41fc61SNicholas Piggin ____##func (regs); \
54425b7e6bbSNicholas Piggin \
545973e2e64SChristophe Leroy interrupt_async_exit_prepare(regs); \
5468d41fc61SNicholas Piggin } \
547e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \
5488d41fc61SNicholas Piggin \
5498d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs)
5508d41fc61SNicholas Piggin
5518d41fc61SNicholas Piggin /**
5528d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function
5538d41fc61SNicholas Piggin * @func: Function name of the entry point
5548d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
5558d41fc61SNicholas Piggin */
5568d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_NMI(func) \
5578d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs)
5588d41fc61SNicholas Piggin
5598d41fc61SNicholas Piggin /**
5608d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function
5618d41fc61SNicholas Piggin * @func: Function name of the entry point
5628d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller
5638d41fc61SNicholas Piggin *
5648d41fc61SNicholas Piggin * @func is called from ASM entry code.
5658d41fc61SNicholas Piggin *
5668d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the
5678d41fc61SNicholas Piggin * body with a pair of curly brackets.
5688d41fc61SNicholas Piggin */
5698d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_NMI(func) \
5705352090aSDaniel Axtens static __always_inline __no_sanitize_address __no_kcsan long \
5715352090aSDaniel Axtens ____##func(struct pt_regs *regs); \
5728d41fc61SNicholas Piggin \
573e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \
5748d41fc61SNicholas Piggin { \
57525b7e6bbSNicholas Piggin struct interrupt_nmi_state state; \
5768d41fc61SNicholas Piggin long ret; \
5778d41fc61SNicholas Piggin \
57825b7e6bbSNicholas Piggin interrupt_nmi_enter_prepare(regs, &state); \
57925b7e6bbSNicholas Piggin \
5808d41fc61SNicholas Piggin ret = ____##func (regs); \
5818d41fc61SNicholas Piggin \
58225b7e6bbSNicholas Piggin interrupt_nmi_exit_prepare(regs, &state); \
58325b7e6bbSNicholas Piggin \
5848d41fc61SNicholas Piggin return ret; \
5858d41fc61SNicholas Piggin } \
586e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \
5878d41fc61SNicholas Piggin \
5885352090aSDaniel Axtens static __always_inline __no_sanitize_address __no_kcsan long \
5895352090aSDaniel Axtens ____##func(struct pt_regs *regs)
5908d41fc61SNicholas Piggin
5913a96570fSNicholas Piggin
5923a96570fSNicholas Piggin /* Interrupt handlers */
5933a96570fSNicholas Piggin /* kernel/traps.c */
5943a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception);
5953a96570fSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64
5962e7ec190SMichael Ellerman DECLARE_INTERRUPT_HANDLER_RAW(machine_check_early_boot);
597f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async);
5983a96570fSNicholas Piggin #endif
599f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception);
6003a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SMIException);
6013a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(handle_hmi_exception);
6023a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(unknown_exception);
6033a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception);
6043db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception);
6053a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception);
6063a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(RunModeException);
6073a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(single_step_exception);
6083a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(program_check_exception);
6093a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt);
6103a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(alignment_exception);
6113a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(StackOverflow);
6123a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(stack_overflow_exception);
6133a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception);
6143a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception);
6153a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception);
6163a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception);
6173a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm);
6183a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm);
6193a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm);
6203a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi);
6213a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async);
6223a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception);
6233a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(DebugException);
6243a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_assist_exception);
6253a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(CacheLockingException);
6263a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException);
6273a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException);
6283db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException);
6293a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_bad_stack);
6303a96570fSNicholas Piggin
6313a96570fSNicholas Piggin /* slb.c */
6323a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault);
633935b534cSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt);
6343a96570fSNicholas Piggin
6353a96570fSNicholas Piggin /* hash_utils.c */
6368b91cee5SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_hash_fault);
6373a96570fSNicholas Piggin
6383a96570fSNicholas Piggin /* fault.c */
639c45ba4f4SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_page_fault);
6403a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv);
6413a96570fSNicholas Piggin
6423a96570fSNicholas Piggin /* process.c */
6433a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_break);
6443a96570fSNicholas Piggin
6453a96570fSNicholas Piggin /* time.c */
6463a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt);
6473a96570fSNicholas Piggin
6483a96570fSNicholas Piggin /* mce.c */
6493a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early);
6503a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode);
6513a96570fSNicholas Piggin
6523a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException);
6533a96570fSNicholas Piggin
65498694166SChristophe Leroy /* irq.c */
65598694166SChristophe Leroy DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ);
65698694166SChristophe Leroy
657a58cbed6SChristophe Leroy void __noreturn unrecoverable_exception(struct pt_regs *regs);
6580b736881SChristophe Leroy
6593a96570fSNicholas Piggin void replay_system_reset(void);
6603a96570fSNicholas Piggin void replay_soft_interrupts(void);
6613a96570fSNicholas Piggin
interrupt_cond_local_irq_enable(struct pt_regs * regs)662e6f8a6c8SNicholas Piggin static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs)
663e6f8a6c8SNicholas Piggin {
664e6f8a6c8SNicholas Piggin if (!arch_irq_disabled_regs(regs))
665e6f8a6c8SNicholas Piggin local_irq_enable();
666e6f8a6c8SNicholas Piggin }
667e6f8a6c8SNicholas Piggin
668f8971c62SRohan McLure long system_call_exception(struct pt_regs *regs, unsigned long r0);
66976222808SChristophe Leroy notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
67076222808SChristophe Leroy notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs);
67176222808SChristophe Leroy notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs);
67276222808SChristophe Leroy #ifdef CONFIG_PPC64
67376222808SChristophe Leroy unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs);
67476222808SChristophe Leroy unsigned long interrupt_exit_user_restart(struct pt_regs *regs);
67576222808SChristophe Leroy unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs);
67676222808SChristophe Leroy #endif
67776222808SChristophe Leroy
6780f5eb28aSChristophe Leroy #endif /* __ASSEMBLY__ */
6790f5eb28aSChristophe Leroy
6808d41fc61SNicholas Piggin #endif /* _ASM_POWERPC_INTERRUPT_H */
681