18d41fc61SNicholas Piggin /* SPDX-License-Identifier: GPL-2.0-or-later */ 28d41fc61SNicholas Piggin #ifndef _ASM_POWERPC_INTERRUPT_H 38d41fc61SNicholas Piggin #define _ASM_POWERPC_INTERRUPT_H 48d41fc61SNicholas Piggin 57153d4bfSXiongwei Song /* BookE/4xx */ 67153d4bfSXiongwei Song #define INTERRUPT_CRITICAL_INPUT 0x100 77153d4bfSXiongwei Song 87153d4bfSXiongwei Song /* BookE */ 97153d4bfSXiongwei Song #define INTERRUPT_DEBUG 0xd00 107153d4bfSXiongwei Song #ifdef CONFIG_BOOKE 117153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x260 127153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0x280 137153d4bfSXiongwei Song #endif 147153d4bfSXiongwei Song 157153d4bfSXiongwei Song /* BookS/4xx/8xx */ 167153d4bfSXiongwei Song #define INTERRUPT_MACHINE_CHECK 0x200 177153d4bfSXiongwei Song 187153d4bfSXiongwei Song /* BookS/8xx */ 197153d4bfSXiongwei Song #define INTERRUPT_SYSTEM_RESET 0x100 207153d4bfSXiongwei Song 217153d4bfSXiongwei Song /* BookS */ 227153d4bfSXiongwei Song #define INTERRUPT_DATA_SEGMENT 0x380 237153d4bfSXiongwei Song #define INTERRUPT_INST_SEGMENT 0x480 247153d4bfSXiongwei Song #define INTERRUPT_TRACE 0xd00 257153d4bfSXiongwei Song #define INTERRUPT_H_DATA_STORAGE 0xe00 26e5223311SChristophe Leroy #define INTERRUPT_HMI 0xe60 277153d4bfSXiongwei Song #define INTERRUPT_H_FAC_UNAVAIL 0xf80 287153d4bfSXiongwei Song #ifdef CONFIG_PPC_BOOK3S 297153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0xa00 307153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0xf00 317fab6397SChristophe Leroy #define INTERRUPT_ALTIVEC_UNAVAIL 0xf20 327153d4bfSXiongwei Song #endif 337153d4bfSXiongwei Song 347153d4bfSXiongwei Song /* BookE/BookS/4xx/8xx */ 357153d4bfSXiongwei Song #define INTERRUPT_DATA_STORAGE 0x300 367153d4bfSXiongwei Song #define INTERRUPT_INST_STORAGE 0x400 370f5eb28aSChristophe Leroy #define INTERRUPT_EXTERNAL 0x500 387153d4bfSXiongwei Song #define INTERRUPT_ALIGNMENT 0x600 397153d4bfSXiongwei Song #define INTERRUPT_PROGRAM 0x700 407153d4bfSXiongwei Song #define INTERRUPT_SYSCALL 0xc00 410f5eb28aSChristophe Leroy #define INTERRUPT_TRACE 0xd00 427153d4bfSXiongwei Song 437153d4bfSXiongwei Song /* BookE/BookS/44x */ 447153d4bfSXiongwei Song #define INTERRUPT_FP_UNAVAIL 0x800 457153d4bfSXiongwei Song 467153d4bfSXiongwei Song /* BookE/BookS/44x/8xx */ 477153d4bfSXiongwei Song #define INTERRUPT_DECREMENTER 0x900 487153d4bfSXiongwei Song 497153d4bfSXiongwei Song #ifndef INTERRUPT_PERFMON 507153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x0 517153d4bfSXiongwei Song #endif 527153d4bfSXiongwei Song 530f5eb28aSChristophe Leroy /* 8xx */ 540f5eb28aSChristophe Leroy #define INTERRUPT_SOFT_EMU_8xx 0x1000 550f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_MISS_8xx 0x1100 560f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_MISS_8xx 0x1200 570f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_ERROR_8xx 0x1300 580f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400 590f5eb28aSChristophe Leroy #define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00 600f5eb28aSChristophe Leroy #define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00 610f5eb28aSChristophe Leroy 627fab6397SChristophe Leroy /* 603 */ 637fab6397SChristophe Leroy #define INTERRUPT_INST_TLB_MISS_603 0x1000 647fab6397SChristophe Leroy #define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100 657fab6397SChristophe Leroy #define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200 667fab6397SChristophe Leroy 670f5eb28aSChristophe Leroy #ifndef __ASSEMBLY__ 680f5eb28aSChristophe Leroy 690f5eb28aSChristophe Leroy #include <linux/context_tracking.h> 700f5eb28aSChristophe Leroy #include <linux/hardirq.h> 710f5eb28aSChristophe Leroy #include <asm/cputime.h> 720f5eb28aSChristophe Leroy #include <asm/ftrace.h> 730f5eb28aSChristophe Leroy #include <asm/kprobes.h> 740f5eb28aSChristophe Leroy #include <asm/runlatch.h> 750f5eb28aSChristophe Leroy 769b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 77f23699c9SNicholas Piggin extern char __end_soft_masked[]; 78325678fdSNicholas Piggin bool search_kernel_soft_mask_table(unsigned long addr); 79f23699c9SNicholas Piggin unsigned long search_kernel_restart_table(unsigned long addr); 80f23699c9SNicholas Piggin 8113799748SNicholas Piggin DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant); 8213799748SNicholas Piggin 839b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs) 849b69d48cSNicholas Piggin { 859b69d48cSNicholas Piggin if (regs->msr & MSR_PR) 869b69d48cSNicholas Piggin return false; 879b69d48cSNicholas Piggin 889b69d48cSNicholas Piggin if (regs->nip >= (unsigned long)__end_soft_masked) 899b69d48cSNicholas Piggin return false; 909b69d48cSNicholas Piggin 91325678fdSNicholas Piggin return search_kernel_soft_mask_table(regs->nip); 929b69d48cSNicholas Piggin } 939b69d48cSNicholas Piggin 9459dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void) 9559dc5bfcSNicholas Piggin { 9659dc5bfcSNicholas Piggin local_paca->srr_valid = 0; 9759dc5bfcSNicholas Piggin local_paca->hsrr_valid = 0; 9859dc5bfcSNicholas Piggin } 9959dc5bfcSNicholas Piggin #else 100ecb1057cSNicholas Piggin static inline unsigned long search_kernel_restart_table(unsigned long addr) 101ecb1057cSNicholas Piggin { 102ecb1057cSNicholas Piggin return 0; 103ecb1057cSNicholas Piggin } 104ecb1057cSNicholas Piggin 1059b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs) 1069b69d48cSNicholas Piggin { 1079b69d48cSNicholas Piggin return false; 1089b69d48cSNicholas Piggin } 1099b69d48cSNicholas Piggin 11059dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void) 11159dc5bfcSNicholas Piggin { 11259dc5bfcSNicholas Piggin } 11359dc5bfcSNicholas Piggin #endif 11459dc5bfcSNicholas Piggin 11598db179aSNicholas Piggin static inline void nap_adjust_return(struct pt_regs *regs) 11698db179aSNicholas Piggin { 11798db179aSNicholas Piggin #ifdef CONFIG_PPC_970_NAP 11898db179aSNicholas Piggin if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { 11998db179aSNicholas Piggin /* Can avoid a test-and-clear because NMIs do not call this */ 12098db179aSNicholas Piggin clear_thread_local_flags(_TLF_NAPPING); 12159dc5bfcSNicholas Piggin regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return); 12298db179aSNicholas Piggin } 12398db179aSNicholas Piggin #endif 12498db179aSNicholas Piggin } 12598db179aSNicholas Piggin 12625b7e6bbSNicholas Piggin struct interrupt_state { 12725b7e6bbSNicholas Piggin }; 12825b7e6bbSNicholas Piggin 129d524dda7SChristophe Leroy static inline void booke_restore_dbcr0(void) 130d524dda7SChristophe Leroy { 131d524dda7SChristophe Leroy #ifdef CONFIG_PPC_ADV_DEBUG_REGS 132d524dda7SChristophe Leroy unsigned long dbcr0 = current->thread.debug.dbcr0; 133d524dda7SChristophe Leroy 134d524dda7SChristophe Leroy if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { 135d524dda7SChristophe Leroy mtspr(SPRN_DBSR, -1); 136d524dda7SChristophe Leroy mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); 137d524dda7SChristophe Leroy } 138d524dda7SChristophe Leroy #endif 139d524dda7SChristophe Leroy } 140d524dda7SChristophe Leroy 14125b7e6bbSNicholas Piggin static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) 14225b7e6bbSNicholas Piggin { 143be39e105SChristophe Leroy #ifdef CONFIG_PPC32 144be39e105SChristophe Leroy if (!arch_irq_disabled_regs(regs)) 145be39e105SChristophe Leroy trace_hardirqs_off(); 146f93d866eSChristophe Leroy 147526d4a4cSChristophe Leroy if (user_mode(regs)) 148937fb700SChristophe Leroy kuap_lock(); 149526d4a4cSChristophe Leroy else 150c1672883SChristophe Leroy kuap_save_and_lock(regs); 151937fb700SChristophe Leroy 152937fb700SChristophe Leroy if (user_mode(regs)) 153937fb700SChristophe Leroy account_cpu_user_entry(); 154be39e105SChristophe Leroy #endif 155097157e1SNicholas Piggin 156097157e1SNicholas Piggin #ifdef CONFIG_PPC64 157ff0b0d6eSNicholas Piggin bool trace_enable = false; 158ff0b0d6eSNicholas Piggin 159ff0b0d6eSNicholas Piggin if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS)) { 16075b96950SNicholas Piggin if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) 161ff0b0d6eSNicholas Piggin trace_enable = true; 162ff0b0d6eSNicholas Piggin } else { 163ff0b0d6eSNicholas Piggin irq_soft_mask_set(IRQS_ALL_DISABLED); 164ff0b0d6eSNicholas Piggin } 1654423eb5aSNicholas Piggin 1664423eb5aSNicholas Piggin /* 1674423eb5aSNicholas Piggin * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE]. 1684423eb5aSNicholas Piggin * Asynchronous interrupts get here with HARD_DIS set (see below), so 1694423eb5aSNicholas Piggin * this enables MSR[EE] for synchronous interrupts. IRQs remain 1704423eb5aSNicholas Piggin * soft-masked. The interrupt handler may later call 1714423eb5aSNicholas Piggin * interrupt_cond_local_irq_enable() to achieve a regular process 1724423eb5aSNicholas Piggin * context. 1734423eb5aSNicholas Piggin */ 1744423eb5aSNicholas Piggin if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) { 1754423eb5aSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 1764423eb5aSNicholas Piggin BUG_ON(!(regs->msr & MSR_EE)); 1774423eb5aSNicholas Piggin __hard_irq_enable(); 178ff0b0d6eSNicholas Piggin } else { 179ff0b0d6eSNicholas Piggin __hard_RI_enable(); 1804423eb5aSNicholas Piggin } 18175b96950SNicholas Piggin 182ff0b0d6eSNicholas Piggin /* Do this when RI=1 because it can cause SLB faults */ 183ff0b0d6eSNicholas Piggin if (trace_enable) 184ff0b0d6eSNicholas Piggin trace_hardirqs_off(); 185ff0b0d6eSNicholas Piggin 186f821bc97SNicholas Piggin if (user_mode(regs)) { 18742e03bc5SChristophe Leroy kuap_lock(); 188f821bc97SNicholas Piggin CT_WARN_ON(ct_state() != CONTEXT_USER); 189f821bc97SNicholas Piggin user_exit_irqoff(); 19056acfdd8SNicholas Piggin 19156acfdd8SNicholas Piggin account_cpu_user_entry(); 19256acfdd8SNicholas Piggin account_stolen_time(); 193f821bc97SNicholas Piggin } else { 19442e03bc5SChristophe Leroy kuap_save_and_lock(regs); 195f821bc97SNicholas Piggin /* 196f821bc97SNicholas Piggin * CT_WARN_ON comes here via program_check_exception, 197f821bc97SNicholas Piggin * so avoid recursion. 198f821bc97SNicholas Piggin */ 1999d1988caSNicholas Piggin if (TRAP(regs) != INTERRUPT_PROGRAM) { 200f821bc97SNicholas Piggin CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 201ecb1057cSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 2029b69d48cSNicholas Piggin BUG_ON(is_implicit_soft_masked(regs)); 2039d1988caSNicholas Piggin } 204ecb1057cSNicholas Piggin 2059d1988caSNicholas Piggin /* Move this under a debugging check */ 206ecb1057cSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && 207ecb1057cSNicholas Piggin arch_irq_disabled_regs(regs)) 2089d1988caSNicholas Piggin BUG_ON(search_kernel_restart_table(regs->nip)); 209f821bc97SNicholas Piggin } 2102b43dd76SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 2112b43dd76SNicholas Piggin BUG_ON(!arch_irq_disabled_regs(regs) && !(regs->msr & MSR_EE)); 212f821bc97SNicholas Piggin #endif 21375b96950SNicholas Piggin 21479f4bb17SChristophe Leroy booke_restore_dbcr0(); 21525b7e6bbSNicholas Piggin } 21625b7e6bbSNicholas Piggin 21725b7e6bbSNicholas Piggin /* 21825b7e6bbSNicholas Piggin * Care should be taken to note that interrupt_exit_prepare and 21925b7e6bbSNicholas Piggin * interrupt_async_exit_prepare do not necessarily return immediately to 22025b7e6bbSNicholas Piggin * regs context (e.g., if regs is usermode, we don't necessarily return to 22125b7e6bbSNicholas Piggin * user mode). Other interrupts might be taken between here and return, 22225b7e6bbSNicholas Piggin * context switch / preemption may occur in the exit path after this, or a 22325b7e6bbSNicholas Piggin * signal may be delivered, etc. 22425b7e6bbSNicholas Piggin * 22525b7e6bbSNicholas Piggin * The real interrupt exit code is platform specific, e.g., 22625b7e6bbSNicholas Piggin * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. 22725b7e6bbSNicholas Piggin * 22825b7e6bbSNicholas Piggin * However interrupt_nmi_exit_prepare does return directly to regs, because 22925b7e6bbSNicholas Piggin * NMIs do not do "exit work" or replay soft-masked interrupts. 23025b7e6bbSNicholas Piggin */ 23125b7e6bbSNicholas Piggin static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 23225b7e6bbSNicholas Piggin { 23325b7e6bbSNicholas Piggin } 23425b7e6bbSNicholas Piggin 23525b7e6bbSNicholas Piggin static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) 23625b7e6bbSNicholas Piggin { 2374423eb5aSNicholas Piggin #ifdef CONFIG_PPC64 2384423eb5aSNicholas Piggin /* Ensure interrupt_enter_prepare does not enable MSR[EE] */ 2394423eb5aSNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 2404423eb5aSNicholas Piggin #endif 241ff0b0d6eSNicholas Piggin interrupt_enter_prepare(regs, state); 24286dbb394SNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 243ff0b0d6eSNicholas Piggin /* 244ff0b0d6eSNicholas Piggin * RI=1 is set by interrupt_enter_prepare, so this thread flags access 245ff0b0d6eSNicholas Piggin * has to come afterward (it can cause SLB faults). 246ff0b0d6eSNicholas Piggin */ 24786dbb394SNicholas Piggin if (cpu_has_feature(CPU_FTR_CTRL) && 24886dbb394SNicholas Piggin !test_thread_local_flags(_TLF_RUNLATCH)) 24986dbb394SNicholas Piggin __ppc64_runlatch_on(); 25086dbb394SNicholas Piggin #endif 2511b1b6a6fSNicholas Piggin irq_enter(); 25225b7e6bbSNicholas Piggin } 25325b7e6bbSNicholas Piggin 25425b7e6bbSNicholas Piggin static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) 25525b7e6bbSNicholas Piggin { 25698db179aSNicholas Piggin /* 25798db179aSNicholas Piggin * Adjust at exit so the main handler sees the true NIA. This must 25898db179aSNicholas Piggin * come before irq_exit() because irq_exit can enable interrupts, and 25998db179aSNicholas Piggin * if another interrupt is taken before nap_adjust_return has run 26098db179aSNicholas Piggin * here, then that interrupt would return directly to idle nap return. 26198db179aSNicholas Piggin */ 26298db179aSNicholas Piggin nap_adjust_return(regs); 26398db179aSNicholas Piggin 2641b1b6a6fSNicholas Piggin irq_exit(); 2656fdb0f41SNicholas Piggin interrupt_exit_prepare(regs, state); 26625b7e6bbSNicholas Piggin } 26725b7e6bbSNicholas Piggin 26825b7e6bbSNicholas Piggin struct interrupt_nmi_state { 269118178e6SNicholas Piggin #ifdef CONFIG_PPC64 2706ecbb582SNicholas Piggin u8 irq_soft_mask; 2716ecbb582SNicholas Piggin u8 irq_happened; 272118178e6SNicholas Piggin u8 ftrace_enabled; 2731b048222SNicholas Piggin u64 softe; 274118178e6SNicholas Piggin #endif 27525b7e6bbSNicholas Piggin }; 27625b7e6bbSNicholas Piggin 2773db8aa10SNicholas Piggin static inline bool nmi_disables_ftrace(struct pt_regs *regs) 2783db8aa10SNicholas Piggin { 2793db8aa10SNicholas Piggin /* Allow DEC and PMI to be traced when they are soft-NMI */ 2803db8aa10SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { 2817153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_DECREMENTER) 2823db8aa10SNicholas Piggin return false; 2837153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON) 2843db8aa10SNicholas Piggin return false; 2853db8aa10SNicholas Piggin } 2863db8aa10SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { 2877153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON) 2883db8aa10SNicholas Piggin return false; 2893db8aa10SNicholas Piggin } 2903db8aa10SNicholas Piggin 2913db8aa10SNicholas Piggin return true; 2923db8aa10SNicholas Piggin } 2933db8aa10SNicholas Piggin 29425b7e6bbSNicholas Piggin static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 29525b7e6bbSNicholas Piggin { 296118178e6SNicholas Piggin #ifdef CONFIG_PPC64 2976ecbb582SNicholas Piggin state->irq_soft_mask = local_paca->irq_soft_mask; 2986ecbb582SNicholas Piggin state->irq_happened = local_paca->irq_happened; 2991b048222SNicholas Piggin state->softe = regs->softe; 3006ecbb582SNicholas Piggin 3016ecbb582SNicholas Piggin /* 3026ecbb582SNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does 3036ecbb582SNicholas Piggin * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile 3046ecbb582SNicholas Piggin * because that goes through irq tracing which we don't want in NMI. 3056ecbb582SNicholas Piggin */ 3066ecbb582SNicholas Piggin local_paca->irq_soft_mask = IRQS_ALL_DISABLED; 3076ecbb582SNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 3086ecbb582SNicholas Piggin 309768c4701SNicholas Piggin if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) { 310768c4701SNicholas Piggin /* 311768c4701SNicholas Piggin * Adjust regs->softe to be soft-masked if it had not been 312768c4701SNicholas Piggin * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe 313768c4701SNicholas Piggin * not yet set disabled), or if it was in an implicit soft 314768c4701SNicholas Piggin * masked state. This makes arch_irq_disabled_regs(regs) 315768c4701SNicholas Piggin * behave as expected. 316768c4701SNicholas Piggin */ 3174ec5feecSNicholas Piggin regs->softe = IRQS_ALL_DISABLED; 3184ec5feecSNicholas Piggin } 3194ec5feecSNicholas Piggin 320ff0b0d6eSNicholas Piggin __hard_RI_enable(); 321ff0b0d6eSNicholas Piggin 3226ecbb582SNicholas Piggin /* Don't do any per-CPU operations until interrupt state is fixed */ 3233db8aa10SNicholas Piggin 3243db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs)) { 325118178e6SNicholas Piggin state->ftrace_enabled = this_cpu_get_ftrace_enabled(); 326118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(0); 327118178e6SNicholas Piggin } 328118178e6SNicholas Piggin #endif 329118178e6SNicholas Piggin 330118178e6SNicholas Piggin /* 331118178e6SNicholas Piggin * Do not use nmi_enter() for pseries hash guest taking a real-mode 332118178e6SNicholas Piggin * NMI because not everything it touches is within the RMA limit. 333118178e6SNicholas Piggin */ 334118178e6SNicholas Piggin if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 335118178e6SNicholas Piggin !firmware_has_feature(FW_FEATURE_LPAR) || 336118178e6SNicholas Piggin radix_enabled() || (mfmsr() & MSR_DR)) 337118178e6SNicholas Piggin nmi_enter(); 33825b7e6bbSNicholas Piggin } 33925b7e6bbSNicholas Piggin 34025b7e6bbSNicholas Piggin static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 34125b7e6bbSNicholas Piggin { 342118178e6SNicholas Piggin if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 343118178e6SNicholas Piggin !firmware_has_feature(FW_FEATURE_LPAR) || 344118178e6SNicholas Piggin radix_enabled() || (mfmsr() & MSR_DR)) 345118178e6SNicholas Piggin nmi_exit(); 346118178e6SNicholas Piggin 34798db179aSNicholas Piggin /* 34898db179aSNicholas Piggin * nmi does not call nap_adjust_return because nmi should not create 34998db179aSNicholas Piggin * new work to do (must use irq_work for that). 35098db179aSNicholas Piggin */ 35198db179aSNicholas Piggin 352118178e6SNicholas Piggin #ifdef CONFIG_PPC64 3539b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S 354f23699c9SNicholas Piggin if (arch_irq_disabled_regs(regs)) { 355f23699c9SNicholas Piggin unsigned long rst = search_kernel_restart_table(regs->nip); 356f23699c9SNicholas Piggin if (rst) 357f23699c9SNicholas Piggin regs_set_return_ip(regs, rst); 358f23699c9SNicholas Piggin } 359f23699c9SNicholas Piggin #endif 360f23699c9SNicholas Piggin 3613db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs)) 362118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(state->ftrace_enabled); 3636ecbb582SNicholas Piggin 3646ecbb582SNicholas Piggin /* Check we didn't change the pending interrupt mask. */ 3656ecbb582SNicholas Piggin WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); 3661b048222SNicholas Piggin regs->softe = state->softe; 3676ecbb582SNicholas Piggin local_paca->irq_happened = state->irq_happened; 3686ecbb582SNicholas Piggin local_paca->irq_soft_mask = state->irq_soft_mask; 3696ecbb582SNicholas Piggin #endif 37025b7e6bbSNicholas Piggin } 37125b7e6bbSNicholas Piggin 372e4bb64c7SNicholas Piggin /* 373e4bb64c7SNicholas Piggin * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each 374e4bb64c7SNicholas Piggin * function definition. The reason for this is the noinstr section is placed 375e4bb64c7SNicholas Piggin * after the main text section, i.e., very far away from the interrupt entry 376e4bb64c7SNicholas Piggin * asm. That creates problems with fitting linker stubs when building large 377e4bb64c7SNicholas Piggin * kernels. 378e4bb64c7SNicholas Piggin */ 379e4bb64c7SNicholas Piggin #define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address 380e4bb64c7SNicholas Piggin 3818d41fc61SNicholas Piggin /** 3828d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function 3838d41fc61SNicholas Piggin * @func: Function name of the entry point 3848d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 3858d41fc61SNicholas Piggin */ 3868d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RAW(func) \ 3878d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 3888d41fc61SNicholas Piggin 3898d41fc61SNicholas Piggin /** 3908d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function 3918d41fc61SNicholas Piggin * @func: Function name of the entry point 3928d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 3938d41fc61SNicholas Piggin * 3948d41fc61SNicholas Piggin * @func is called from ASM entry code. 3958d41fc61SNicholas Piggin * 3968d41fc61SNicholas Piggin * This is a plain function which does no tracing, reconciling, etc. 3978d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 3988d41fc61SNicholas Piggin * body with a pair of curly brackets. 3998d41fc61SNicholas Piggin * 4008d41fc61SNicholas Piggin * raw interrupt handlers must not enable or disable interrupts, or 4018d41fc61SNicholas Piggin * schedule, tracing and instrumentation (ftrace, lockdep, etc) would 4028d41fc61SNicholas Piggin * not be advisable either, although may be possible in a pinch, the 4038d41fc61SNicholas Piggin * trace will look odd at least. 4048d41fc61SNicholas Piggin * 4058d41fc61SNicholas Piggin * A raw handler may call one of the other interrupt handler functions 4068d41fc61SNicholas Piggin * to be converted into that interrupt context without these restrictions. 4078d41fc61SNicholas Piggin * 4088d41fc61SNicholas Piggin * On PPC64, _RAW handlers may return with fast_interrupt_return. 4098d41fc61SNicholas Piggin * 4108d41fc61SNicholas Piggin * Specific handlers may have additional restrictions. 4118d41fc61SNicholas Piggin */ 4128d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RAW(func) \ 4138d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 4148d41fc61SNicholas Piggin \ 415e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 4168d41fc61SNicholas Piggin { \ 4178d41fc61SNicholas Piggin long ret; \ 4188d41fc61SNicholas Piggin \ 419ff0b0d6eSNicholas Piggin __hard_RI_enable(); \ 420ff0b0d6eSNicholas Piggin \ 4218d41fc61SNicholas Piggin ret = ____##func (regs); \ 4228d41fc61SNicholas Piggin \ 4238d41fc61SNicholas Piggin return ret; \ 4248d41fc61SNicholas Piggin } \ 425e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4268d41fc61SNicholas Piggin \ 4278d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 4288d41fc61SNicholas Piggin 4298d41fc61SNicholas Piggin /** 4308d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function 4318d41fc61SNicholas Piggin * @func: Function name of the entry point 4328d41fc61SNicholas Piggin */ 4338d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER(func) \ 4348d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs) 4358d41fc61SNicholas Piggin 4368d41fc61SNicholas Piggin /** 4378d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function 4388d41fc61SNicholas Piggin * @func: Function name of the entry point 4398d41fc61SNicholas Piggin * 4408d41fc61SNicholas Piggin * @func is called from ASM entry code. 4418d41fc61SNicholas Piggin * 4428d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 4438d41fc61SNicholas Piggin * body with a pair of curly brackets. 4448d41fc61SNicholas Piggin */ 4458d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER(func) \ 4468d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \ 4478d41fc61SNicholas Piggin \ 448e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \ 4498d41fc61SNicholas Piggin { \ 45025b7e6bbSNicholas Piggin struct interrupt_state state; \ 45125b7e6bbSNicholas Piggin \ 45225b7e6bbSNicholas Piggin interrupt_enter_prepare(regs, &state); \ 45325b7e6bbSNicholas Piggin \ 4548d41fc61SNicholas Piggin ____##func (regs); \ 45525b7e6bbSNicholas Piggin \ 45625b7e6bbSNicholas Piggin interrupt_exit_prepare(regs, &state); \ 4578d41fc61SNicholas Piggin } \ 458e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4598d41fc61SNicholas Piggin \ 4608d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs) 4618d41fc61SNicholas Piggin 4628d41fc61SNicholas Piggin /** 4638d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function 4648d41fc61SNicholas Piggin * @func: Function name of the entry point 4658d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 4668d41fc61SNicholas Piggin */ 4678d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RET(func) \ 4688d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 4698d41fc61SNicholas Piggin 4708d41fc61SNicholas Piggin /** 4718d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function 4728d41fc61SNicholas Piggin * @func: Function name of the entry point 4738d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 4748d41fc61SNicholas Piggin * 4758d41fc61SNicholas Piggin * @func is called from ASM entry code. 4768d41fc61SNicholas Piggin * 4778d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 4788d41fc61SNicholas Piggin * body with a pair of curly brackets. 4798d41fc61SNicholas Piggin */ 4808d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RET(func) \ 4818d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 4828d41fc61SNicholas Piggin \ 483e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 4848d41fc61SNicholas Piggin { \ 48525b7e6bbSNicholas Piggin struct interrupt_state state; \ 4868d41fc61SNicholas Piggin long ret; \ 4878d41fc61SNicholas Piggin \ 48825b7e6bbSNicholas Piggin interrupt_enter_prepare(regs, &state); \ 48925b7e6bbSNicholas Piggin \ 4908d41fc61SNicholas Piggin ret = ____##func (regs); \ 4918d41fc61SNicholas Piggin \ 49225b7e6bbSNicholas Piggin interrupt_exit_prepare(regs, &state); \ 49325b7e6bbSNicholas Piggin \ 4948d41fc61SNicholas Piggin return ret; \ 4958d41fc61SNicholas Piggin } \ 496e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4978d41fc61SNicholas Piggin \ 4988d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 4998d41fc61SNicholas Piggin 5008d41fc61SNicholas Piggin /** 5018d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function 5028d41fc61SNicholas Piggin * @func: Function name of the entry point 5038d41fc61SNicholas Piggin */ 5048d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ 5058d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs) 5068d41fc61SNicholas Piggin 5078d41fc61SNicholas Piggin /** 5088d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function 5098d41fc61SNicholas Piggin * @func: Function name of the entry point 5108d41fc61SNicholas Piggin * 5118d41fc61SNicholas Piggin * @func is called from ASM entry code. 5128d41fc61SNicholas Piggin * 5138d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 5148d41fc61SNicholas Piggin * body with a pair of curly brackets. 5158d41fc61SNicholas Piggin */ 5168d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ 5178d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \ 5188d41fc61SNicholas Piggin \ 519e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \ 5208d41fc61SNicholas Piggin { \ 52125b7e6bbSNicholas Piggin struct interrupt_state state; \ 52225b7e6bbSNicholas Piggin \ 52325b7e6bbSNicholas Piggin interrupt_async_enter_prepare(regs, &state); \ 52425b7e6bbSNicholas Piggin \ 5258d41fc61SNicholas Piggin ____##func (regs); \ 52625b7e6bbSNicholas Piggin \ 52725b7e6bbSNicholas Piggin interrupt_async_exit_prepare(regs, &state); \ 5288d41fc61SNicholas Piggin } \ 529e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 5308d41fc61SNicholas Piggin \ 5318d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs) 5328d41fc61SNicholas Piggin 5338d41fc61SNicholas Piggin /** 5348d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function 5358d41fc61SNicholas Piggin * @func: Function name of the entry point 5368d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 5378d41fc61SNicholas Piggin */ 5388d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_NMI(func) \ 5398d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 5408d41fc61SNicholas Piggin 5418d41fc61SNicholas Piggin /** 5428d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function 5438d41fc61SNicholas Piggin * @func: Function name of the entry point 5448d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 5458d41fc61SNicholas Piggin * 5468d41fc61SNicholas Piggin * @func is called from ASM entry code. 5478d41fc61SNicholas Piggin * 5488d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 5498d41fc61SNicholas Piggin * body with a pair of curly brackets. 5508d41fc61SNicholas Piggin */ 5518d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_NMI(func) \ 5528d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 5538d41fc61SNicholas Piggin \ 554e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 5558d41fc61SNicholas Piggin { \ 55625b7e6bbSNicholas Piggin struct interrupt_nmi_state state; \ 5578d41fc61SNicholas Piggin long ret; \ 5588d41fc61SNicholas Piggin \ 55925b7e6bbSNicholas Piggin interrupt_nmi_enter_prepare(regs, &state); \ 56025b7e6bbSNicholas Piggin \ 5618d41fc61SNicholas Piggin ret = ____##func (regs); \ 5628d41fc61SNicholas Piggin \ 56325b7e6bbSNicholas Piggin interrupt_nmi_exit_prepare(regs, &state); \ 56425b7e6bbSNicholas Piggin \ 5658d41fc61SNicholas Piggin return ret; \ 5668d41fc61SNicholas Piggin } \ 567e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 5688d41fc61SNicholas Piggin \ 5698d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 5708d41fc61SNicholas Piggin 5713a96570fSNicholas Piggin 5723a96570fSNicholas Piggin /* Interrupt handlers */ 5733a96570fSNicholas Piggin /* kernel/traps.c */ 5743a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); 5753a96570fSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 576f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async); 5773a96570fSNicholas Piggin #endif 578f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); 5793a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SMIException); 5803a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); 5813a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(unknown_exception); 5823a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); 5833db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception); 5843a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); 5853a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(RunModeException); 5863a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(single_step_exception); 5873a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(program_check_exception); 5883a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); 5893a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(alignment_exception); 5903a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(StackOverflow); 5913a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); 5923a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); 5933a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); 5943a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); 5953a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); 5963a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); 5973a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); 5983a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); 5993a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); 6003a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); 6013a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); 6023a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(DebugException); 6033a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); 6043a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(CacheLockingException); 6053a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); 6063a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); 6073db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException); 6083a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); 6093a96570fSNicholas Piggin 6103a96570fSNicholas Piggin /* slb.c */ 6113a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); 612935b534cSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt); 6133a96570fSNicholas Piggin 6143a96570fSNicholas Piggin /* hash_utils.c */ 615*8b91cee5SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_hash_fault); 6163a96570fSNicholas Piggin 6173a96570fSNicholas Piggin /* fault.c */ 618c45ba4f4SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_page_fault); 6193a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); 6203a96570fSNicholas Piggin 6213a96570fSNicholas Piggin /* process.c */ 6223a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_break); 6233a96570fSNicholas Piggin 6243a96570fSNicholas Piggin /* time.c */ 6253a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); 6263a96570fSNicholas Piggin 6273a96570fSNicholas Piggin /* mce.c */ 6283a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); 6293a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); 6303a96570fSNicholas Piggin 6313a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); 6323a96570fSNicholas Piggin 63398694166SChristophe Leroy /* irq.c */ 63498694166SChristophe Leroy DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ); 63598694166SChristophe Leroy 636a58cbed6SChristophe Leroy void __noreturn unrecoverable_exception(struct pt_regs *regs); 6370b736881SChristophe Leroy 6383a96570fSNicholas Piggin void replay_system_reset(void); 6393a96570fSNicholas Piggin void replay_soft_interrupts(void); 6403a96570fSNicholas Piggin 641e6f8a6c8SNicholas Piggin static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) 642e6f8a6c8SNicholas Piggin { 643e6f8a6c8SNicholas Piggin if (!arch_irq_disabled_regs(regs)) 644e6f8a6c8SNicholas Piggin local_irq_enable(); 645e6f8a6c8SNicholas Piggin } 646e6f8a6c8SNicholas Piggin 6470f5eb28aSChristophe Leroy #endif /* __ASSEMBLY__ */ 6480f5eb28aSChristophe Leroy 6498d41fc61SNicholas Piggin #endif /* _ASM_POWERPC_INTERRUPT_H */ 650