18d41fc61SNicholas Piggin /* SPDX-License-Identifier: GPL-2.0-or-later */ 28d41fc61SNicholas Piggin #ifndef _ASM_POWERPC_INTERRUPT_H 38d41fc61SNicholas Piggin #define _ASM_POWERPC_INTERRUPT_H 48d41fc61SNicholas Piggin 57153d4bfSXiongwei Song /* BookE/4xx */ 67153d4bfSXiongwei Song #define INTERRUPT_CRITICAL_INPUT 0x100 77153d4bfSXiongwei Song 87153d4bfSXiongwei Song /* BookE */ 97153d4bfSXiongwei Song #define INTERRUPT_DEBUG 0xd00 107153d4bfSXiongwei Song #ifdef CONFIG_BOOKE 117153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x260 127153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0x280 137153d4bfSXiongwei Song #endif 147153d4bfSXiongwei Song 157153d4bfSXiongwei Song /* BookS/4xx/8xx */ 167153d4bfSXiongwei Song #define INTERRUPT_MACHINE_CHECK 0x200 177153d4bfSXiongwei Song 187153d4bfSXiongwei Song /* BookS/8xx */ 197153d4bfSXiongwei Song #define INTERRUPT_SYSTEM_RESET 0x100 207153d4bfSXiongwei Song 217153d4bfSXiongwei Song /* BookS */ 227153d4bfSXiongwei Song #define INTERRUPT_DATA_SEGMENT 0x380 237153d4bfSXiongwei Song #define INTERRUPT_INST_SEGMENT 0x480 247153d4bfSXiongwei Song #define INTERRUPT_TRACE 0xd00 257153d4bfSXiongwei Song #define INTERRUPT_H_DATA_STORAGE 0xe00 26e5223311SChristophe Leroy #define INTERRUPT_HMI 0xe60 277153d4bfSXiongwei Song #define INTERRUPT_H_FAC_UNAVAIL 0xf80 287153d4bfSXiongwei Song #ifdef CONFIG_PPC_BOOK3S 297153d4bfSXiongwei Song #define INTERRUPT_DOORBELL 0xa00 307153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0xf00 317fab6397SChristophe Leroy #define INTERRUPT_ALTIVEC_UNAVAIL 0xf20 327153d4bfSXiongwei Song #endif 337153d4bfSXiongwei Song 347153d4bfSXiongwei Song /* BookE/BookS/4xx/8xx */ 357153d4bfSXiongwei Song #define INTERRUPT_DATA_STORAGE 0x300 367153d4bfSXiongwei Song #define INTERRUPT_INST_STORAGE 0x400 370f5eb28aSChristophe Leroy #define INTERRUPT_EXTERNAL 0x500 387153d4bfSXiongwei Song #define INTERRUPT_ALIGNMENT 0x600 397153d4bfSXiongwei Song #define INTERRUPT_PROGRAM 0x700 407153d4bfSXiongwei Song #define INTERRUPT_SYSCALL 0xc00 410f5eb28aSChristophe Leroy #define INTERRUPT_TRACE 0xd00 427153d4bfSXiongwei Song 437153d4bfSXiongwei Song /* BookE/BookS/44x */ 447153d4bfSXiongwei Song #define INTERRUPT_FP_UNAVAIL 0x800 457153d4bfSXiongwei Song 467153d4bfSXiongwei Song /* BookE/BookS/44x/8xx */ 477153d4bfSXiongwei Song #define INTERRUPT_DECREMENTER 0x900 487153d4bfSXiongwei Song 497153d4bfSXiongwei Song #ifndef INTERRUPT_PERFMON 507153d4bfSXiongwei Song #define INTERRUPT_PERFMON 0x0 517153d4bfSXiongwei Song #endif 527153d4bfSXiongwei Song 530f5eb28aSChristophe Leroy /* 8xx */ 540f5eb28aSChristophe Leroy #define INTERRUPT_SOFT_EMU_8xx 0x1000 550f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_MISS_8xx 0x1100 560f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_MISS_8xx 0x1200 570f5eb28aSChristophe Leroy #define INTERRUPT_INST_TLB_ERROR_8xx 0x1300 580f5eb28aSChristophe Leroy #define INTERRUPT_DATA_TLB_ERROR_8xx 0x1400 590f5eb28aSChristophe Leroy #define INTERRUPT_DATA_BREAKPOINT_8xx 0x1c00 600f5eb28aSChristophe Leroy #define INTERRUPT_INST_BREAKPOINT_8xx 0x1d00 610f5eb28aSChristophe Leroy 627fab6397SChristophe Leroy /* 603 */ 637fab6397SChristophe Leroy #define INTERRUPT_INST_TLB_MISS_603 0x1000 647fab6397SChristophe Leroy #define INTERRUPT_DATA_LOAD_TLB_MISS_603 0x1100 657fab6397SChristophe Leroy #define INTERRUPT_DATA_STORE_TLB_MISS_603 0x1200 667fab6397SChristophe Leroy 670f5eb28aSChristophe Leroy #ifndef __ASSEMBLY__ 680f5eb28aSChristophe Leroy 690f5eb28aSChristophe Leroy #include <linux/context_tracking.h> 700f5eb28aSChristophe Leroy #include <linux/hardirq.h> 710f5eb28aSChristophe Leroy #include <asm/cputime.h> 720f5eb28aSChristophe Leroy #include <asm/ftrace.h> 730f5eb28aSChristophe Leroy #include <asm/kprobes.h> 740f5eb28aSChristophe Leroy #include <asm/runlatch.h> 750f5eb28aSChristophe Leroy 769b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 77f23699c9SNicholas Piggin extern char __end_soft_masked[]; 78325678fdSNicholas Piggin bool search_kernel_soft_mask_table(unsigned long addr); 79f23699c9SNicholas Piggin unsigned long search_kernel_restart_table(unsigned long addr); 80f23699c9SNicholas Piggin 8113799748SNicholas Piggin DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant); 8213799748SNicholas Piggin 839b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs) 849b69d48cSNicholas Piggin { 859b69d48cSNicholas Piggin if (regs->msr & MSR_PR) 869b69d48cSNicholas Piggin return false; 879b69d48cSNicholas Piggin 889b69d48cSNicholas Piggin if (regs->nip >= (unsigned long)__end_soft_masked) 899b69d48cSNicholas Piggin return false; 909b69d48cSNicholas Piggin 91325678fdSNicholas Piggin return search_kernel_soft_mask_table(regs->nip); 929b69d48cSNicholas Piggin } 939b69d48cSNicholas Piggin 9459dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void) 9559dc5bfcSNicholas Piggin { 9659dc5bfcSNicholas Piggin local_paca->srr_valid = 0; 9759dc5bfcSNicholas Piggin local_paca->hsrr_valid = 0; 9859dc5bfcSNicholas Piggin } 9959dc5bfcSNicholas Piggin #else 100ecb1057cSNicholas Piggin static inline unsigned long search_kernel_restart_table(unsigned long addr) 101ecb1057cSNicholas Piggin { 102ecb1057cSNicholas Piggin return 0; 103ecb1057cSNicholas Piggin } 104ecb1057cSNicholas Piggin 1059b69d48cSNicholas Piggin static inline bool is_implicit_soft_masked(struct pt_regs *regs) 1069b69d48cSNicholas Piggin { 1079b69d48cSNicholas Piggin return false; 1089b69d48cSNicholas Piggin } 1099b69d48cSNicholas Piggin 11059dc5bfcSNicholas Piggin static inline void srr_regs_clobbered(void) 11159dc5bfcSNicholas Piggin { 11259dc5bfcSNicholas Piggin } 11359dc5bfcSNicholas Piggin #endif 11459dc5bfcSNicholas Piggin 11598db179aSNicholas Piggin static inline void nap_adjust_return(struct pt_regs *regs) 11698db179aSNicholas Piggin { 11798db179aSNicholas Piggin #ifdef CONFIG_PPC_970_NAP 11898db179aSNicholas Piggin if (unlikely(test_thread_local_flags(_TLF_NAPPING))) { 11998db179aSNicholas Piggin /* Can avoid a test-and-clear because NMIs do not call this */ 12098db179aSNicholas Piggin clear_thread_local_flags(_TLF_NAPPING); 12159dc5bfcSNicholas Piggin regs_set_return_ip(regs, (unsigned long)power4_idle_nap_return); 12298db179aSNicholas Piggin } 12398db179aSNicholas Piggin #endif 12498db179aSNicholas Piggin } 12598db179aSNicholas Piggin 126d524dda7SChristophe Leroy static inline void booke_restore_dbcr0(void) 127d524dda7SChristophe Leroy { 128d524dda7SChristophe Leroy #ifdef CONFIG_PPC_ADV_DEBUG_REGS 129d524dda7SChristophe Leroy unsigned long dbcr0 = current->thread.debug.dbcr0; 130d524dda7SChristophe Leroy 131d524dda7SChristophe Leroy if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { 132d524dda7SChristophe Leroy mtspr(SPRN_DBSR, -1); 133d524dda7SChristophe Leroy mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); 134d524dda7SChristophe Leroy } 135d524dda7SChristophe Leroy #endif 136d524dda7SChristophe Leroy } 137d524dda7SChristophe Leroy 138973e2e64SChristophe Leroy static inline void interrupt_enter_prepare(struct pt_regs *regs) 13925b7e6bbSNicholas Piggin { 140be39e105SChristophe Leroy #ifdef CONFIG_PPC32 141be39e105SChristophe Leroy if (!arch_irq_disabled_regs(regs)) 142be39e105SChristophe Leroy trace_hardirqs_off(); 143f93d866eSChristophe Leroy 144526d4a4cSChristophe Leroy if (user_mode(regs)) 145937fb700SChristophe Leroy kuap_lock(); 146526d4a4cSChristophe Leroy else 147c1672883SChristophe Leroy kuap_save_and_lock(regs); 148937fb700SChristophe Leroy 149937fb700SChristophe Leroy if (user_mode(regs)) 150937fb700SChristophe Leroy account_cpu_user_entry(); 151be39e105SChristophe Leroy #endif 152097157e1SNicholas Piggin 153097157e1SNicholas Piggin #ifdef CONFIG_PPC64 154ff0b0d6eSNicholas Piggin bool trace_enable = false; 155ff0b0d6eSNicholas Piggin 156ff0b0d6eSNicholas Piggin if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS)) { 15775b96950SNicholas Piggin if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) 158ff0b0d6eSNicholas Piggin trace_enable = true; 159ff0b0d6eSNicholas Piggin } else { 160ff0b0d6eSNicholas Piggin irq_soft_mask_set(IRQS_ALL_DISABLED); 161ff0b0d6eSNicholas Piggin } 1624423eb5aSNicholas Piggin 1634423eb5aSNicholas Piggin /* 1644423eb5aSNicholas Piggin * If the interrupt was taken with HARD_DIS clear, then enable MSR[EE]. 1654423eb5aSNicholas Piggin * Asynchronous interrupts get here with HARD_DIS set (see below), so 1664423eb5aSNicholas Piggin * this enables MSR[EE] for synchronous interrupts. IRQs remain 1674423eb5aSNicholas Piggin * soft-masked. The interrupt handler may later call 1684423eb5aSNicholas Piggin * interrupt_cond_local_irq_enable() to achieve a regular process 1694423eb5aSNicholas Piggin * context. 1704423eb5aSNicholas Piggin */ 1714423eb5aSNicholas Piggin if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS)) { 1724423eb5aSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 1734423eb5aSNicholas Piggin BUG_ON(!(regs->msr & MSR_EE)); 1744423eb5aSNicholas Piggin __hard_irq_enable(); 175ff0b0d6eSNicholas Piggin } else { 176ff0b0d6eSNicholas Piggin __hard_RI_enable(); 1774423eb5aSNicholas Piggin } 17875b96950SNicholas Piggin 179ff0b0d6eSNicholas Piggin /* Do this when RI=1 because it can cause SLB faults */ 180ff0b0d6eSNicholas Piggin if (trace_enable) 181ff0b0d6eSNicholas Piggin trace_hardirqs_off(); 182ff0b0d6eSNicholas Piggin 183f821bc97SNicholas Piggin if (user_mode(regs)) { 18442e03bc5SChristophe Leroy kuap_lock(); 185f821bc97SNicholas Piggin CT_WARN_ON(ct_state() != CONTEXT_USER); 186f821bc97SNicholas Piggin user_exit_irqoff(); 18756acfdd8SNicholas Piggin 18856acfdd8SNicholas Piggin account_cpu_user_entry(); 18956acfdd8SNicholas Piggin account_stolen_time(); 190f821bc97SNicholas Piggin } else { 19142e03bc5SChristophe Leroy kuap_save_and_lock(regs); 192f821bc97SNicholas Piggin /* 193f821bc97SNicholas Piggin * CT_WARN_ON comes here via program_check_exception, 194f821bc97SNicholas Piggin * so avoid recursion. 195f821bc97SNicholas Piggin */ 1969d1988caSNicholas Piggin if (TRAP(regs) != INTERRUPT_PROGRAM) { 197f821bc97SNicholas Piggin CT_WARN_ON(ct_state() != CONTEXT_KERNEL); 198ecb1057cSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 1999b69d48cSNicholas Piggin BUG_ON(is_implicit_soft_masked(regs)); 2009d1988caSNicholas Piggin } 201ecb1057cSNicholas Piggin 2029d1988caSNicholas Piggin /* Move this under a debugging check */ 203ecb1057cSNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && 204ecb1057cSNicholas Piggin arch_irq_disabled_regs(regs)) 2059d1988caSNicholas Piggin BUG_ON(search_kernel_restart_table(regs->nip)); 206f821bc97SNicholas Piggin } 2072b43dd76SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) 2082b43dd76SNicholas Piggin BUG_ON(!arch_irq_disabled_regs(regs) && !(regs->msr & MSR_EE)); 209f821bc97SNicholas Piggin #endif 21075b96950SNicholas Piggin 21179f4bb17SChristophe Leroy booke_restore_dbcr0(); 21225b7e6bbSNicholas Piggin } 21325b7e6bbSNicholas Piggin 21425b7e6bbSNicholas Piggin /* 21525b7e6bbSNicholas Piggin * Care should be taken to note that interrupt_exit_prepare and 21625b7e6bbSNicholas Piggin * interrupt_async_exit_prepare do not necessarily return immediately to 21725b7e6bbSNicholas Piggin * regs context (e.g., if regs is usermode, we don't necessarily return to 21825b7e6bbSNicholas Piggin * user mode). Other interrupts might be taken between here and return, 21925b7e6bbSNicholas Piggin * context switch / preemption may occur in the exit path after this, or a 22025b7e6bbSNicholas Piggin * signal may be delivered, etc. 22125b7e6bbSNicholas Piggin * 22225b7e6bbSNicholas Piggin * The real interrupt exit code is platform specific, e.g., 22325b7e6bbSNicholas Piggin * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. 22425b7e6bbSNicholas Piggin * 22525b7e6bbSNicholas Piggin * However interrupt_nmi_exit_prepare does return directly to regs, because 22625b7e6bbSNicholas Piggin * NMIs do not do "exit work" or replay soft-masked interrupts. 22725b7e6bbSNicholas Piggin */ 228973e2e64SChristophe Leroy static inline void interrupt_exit_prepare(struct pt_regs *regs) 22925b7e6bbSNicholas Piggin { 23025b7e6bbSNicholas Piggin } 23125b7e6bbSNicholas Piggin 232973e2e64SChristophe Leroy static inline void interrupt_async_enter_prepare(struct pt_regs *regs) 23325b7e6bbSNicholas Piggin { 2344423eb5aSNicholas Piggin #ifdef CONFIG_PPC64 2354423eb5aSNicholas Piggin /* Ensure interrupt_enter_prepare does not enable MSR[EE] */ 2364423eb5aSNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 2374423eb5aSNicholas Piggin #endif 238973e2e64SChristophe Leroy interrupt_enter_prepare(regs); 23986dbb394SNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 240ff0b0d6eSNicholas Piggin /* 241ff0b0d6eSNicholas Piggin * RI=1 is set by interrupt_enter_prepare, so this thread flags access 242ff0b0d6eSNicholas Piggin * has to come afterward (it can cause SLB faults). 243ff0b0d6eSNicholas Piggin */ 24486dbb394SNicholas Piggin if (cpu_has_feature(CPU_FTR_CTRL) && 24586dbb394SNicholas Piggin !test_thread_local_flags(_TLF_RUNLATCH)) 24686dbb394SNicholas Piggin __ppc64_runlatch_on(); 24786dbb394SNicholas Piggin #endif 2481b1b6a6fSNicholas Piggin irq_enter(); 24925b7e6bbSNicholas Piggin } 25025b7e6bbSNicholas Piggin 251973e2e64SChristophe Leroy static inline void interrupt_async_exit_prepare(struct pt_regs *regs) 25225b7e6bbSNicholas Piggin { 25398db179aSNicholas Piggin /* 25498db179aSNicholas Piggin * Adjust at exit so the main handler sees the true NIA. This must 25598db179aSNicholas Piggin * come before irq_exit() because irq_exit can enable interrupts, and 25698db179aSNicholas Piggin * if another interrupt is taken before nap_adjust_return has run 25798db179aSNicholas Piggin * here, then that interrupt would return directly to idle nap return. 25898db179aSNicholas Piggin */ 25998db179aSNicholas Piggin nap_adjust_return(regs); 26098db179aSNicholas Piggin 2611b1b6a6fSNicholas Piggin irq_exit(); 262973e2e64SChristophe Leroy interrupt_exit_prepare(regs); 26325b7e6bbSNicholas Piggin } 26425b7e6bbSNicholas Piggin 26525b7e6bbSNicholas Piggin struct interrupt_nmi_state { 266118178e6SNicholas Piggin #ifdef CONFIG_PPC64 2676ecbb582SNicholas Piggin u8 irq_soft_mask; 2686ecbb582SNicholas Piggin u8 irq_happened; 269118178e6SNicholas Piggin u8 ftrace_enabled; 2701b048222SNicholas Piggin u64 softe; 271118178e6SNicholas Piggin #endif 27225b7e6bbSNicholas Piggin }; 27325b7e6bbSNicholas Piggin 2743db8aa10SNicholas Piggin static inline bool nmi_disables_ftrace(struct pt_regs *regs) 2753db8aa10SNicholas Piggin { 2763db8aa10SNicholas Piggin /* Allow DEC and PMI to be traced when they are soft-NMI */ 2773db8aa10SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { 2787153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_DECREMENTER) 2793db8aa10SNicholas Piggin return false; 2807153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON) 2813db8aa10SNicholas Piggin return false; 2823db8aa10SNicholas Piggin } 2833db8aa10SNicholas Piggin if (IS_ENABLED(CONFIG_PPC_BOOK3E)) { 2847153d4bfSXiongwei Song if (TRAP(regs) == INTERRUPT_PERFMON) 2853db8aa10SNicholas Piggin return false; 2863db8aa10SNicholas Piggin } 2873db8aa10SNicholas Piggin 2883db8aa10SNicholas Piggin return true; 2893db8aa10SNicholas Piggin } 2903db8aa10SNicholas Piggin 29125b7e6bbSNicholas Piggin static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 29225b7e6bbSNicholas Piggin { 293118178e6SNicholas Piggin #ifdef CONFIG_PPC64 2946ecbb582SNicholas Piggin state->irq_soft_mask = local_paca->irq_soft_mask; 2956ecbb582SNicholas Piggin state->irq_happened = local_paca->irq_happened; 2961b048222SNicholas Piggin state->softe = regs->softe; 2976ecbb582SNicholas Piggin 2986ecbb582SNicholas Piggin /* 2996ecbb582SNicholas Piggin * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does 3006ecbb582SNicholas Piggin * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile 3016ecbb582SNicholas Piggin * because that goes through irq tracing which we don't want in NMI. 3026ecbb582SNicholas Piggin */ 3036ecbb582SNicholas Piggin local_paca->irq_soft_mask = IRQS_ALL_DISABLED; 3046ecbb582SNicholas Piggin local_paca->irq_happened |= PACA_IRQ_HARD_DIS; 3056ecbb582SNicholas Piggin 306768c4701SNicholas Piggin if (!(regs->msr & MSR_EE) || is_implicit_soft_masked(regs)) { 307768c4701SNicholas Piggin /* 308768c4701SNicholas Piggin * Adjust regs->softe to be soft-masked if it had not been 309768c4701SNicholas Piggin * reconcied (e.g., interrupt entry with MSR[EE]=0 but softe 310768c4701SNicholas Piggin * not yet set disabled), or if it was in an implicit soft 311768c4701SNicholas Piggin * masked state. This makes arch_irq_disabled_regs(regs) 312768c4701SNicholas Piggin * behave as expected. 313768c4701SNicholas Piggin */ 3144ec5feecSNicholas Piggin regs->softe = IRQS_ALL_DISABLED; 3154ec5feecSNicholas Piggin } 3164ec5feecSNicholas Piggin 317ff0b0d6eSNicholas Piggin __hard_RI_enable(); 318ff0b0d6eSNicholas Piggin 3196ecbb582SNicholas Piggin /* Don't do any per-CPU operations until interrupt state is fixed */ 3203db8aa10SNicholas Piggin 3213db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs)) { 322118178e6SNicholas Piggin state->ftrace_enabled = this_cpu_get_ftrace_enabled(); 323118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(0); 324118178e6SNicholas Piggin } 325118178e6SNicholas Piggin #endif 326118178e6SNicholas Piggin 327118178e6SNicholas Piggin /* 328118178e6SNicholas Piggin * Do not use nmi_enter() for pseries hash guest taking a real-mode 329118178e6SNicholas Piggin * NMI because not everything it touches is within the RMA limit. 330118178e6SNicholas Piggin */ 331118178e6SNicholas Piggin if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 332118178e6SNicholas Piggin !firmware_has_feature(FW_FEATURE_LPAR) || 333118178e6SNicholas Piggin radix_enabled() || (mfmsr() & MSR_DR)) 334118178e6SNicholas Piggin nmi_enter(); 33525b7e6bbSNicholas Piggin } 33625b7e6bbSNicholas Piggin 33725b7e6bbSNicholas Piggin static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) 33825b7e6bbSNicholas Piggin { 339118178e6SNicholas Piggin if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || 340118178e6SNicholas Piggin !firmware_has_feature(FW_FEATURE_LPAR) || 341118178e6SNicholas Piggin radix_enabled() || (mfmsr() & MSR_DR)) 342118178e6SNicholas Piggin nmi_exit(); 343118178e6SNicholas Piggin 34498db179aSNicholas Piggin /* 34598db179aSNicholas Piggin * nmi does not call nap_adjust_return because nmi should not create 34698db179aSNicholas Piggin * new work to do (must use irq_work for that). 34798db179aSNicholas Piggin */ 34898db179aSNicholas Piggin 349118178e6SNicholas Piggin #ifdef CONFIG_PPC64 3509b69d48cSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S 351f23699c9SNicholas Piggin if (arch_irq_disabled_regs(regs)) { 352f23699c9SNicholas Piggin unsigned long rst = search_kernel_restart_table(regs->nip); 353f23699c9SNicholas Piggin if (rst) 354f23699c9SNicholas Piggin regs_set_return_ip(regs, rst); 355f23699c9SNicholas Piggin } 356f23699c9SNicholas Piggin #endif 357f23699c9SNicholas Piggin 3583db8aa10SNicholas Piggin if (nmi_disables_ftrace(regs)) 359118178e6SNicholas Piggin this_cpu_set_ftrace_enabled(state->ftrace_enabled); 3606ecbb582SNicholas Piggin 3616ecbb582SNicholas Piggin /* Check we didn't change the pending interrupt mask. */ 3626ecbb582SNicholas Piggin WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); 3631b048222SNicholas Piggin regs->softe = state->softe; 3646ecbb582SNicholas Piggin local_paca->irq_happened = state->irq_happened; 3656ecbb582SNicholas Piggin local_paca->irq_soft_mask = state->irq_soft_mask; 3666ecbb582SNicholas Piggin #endif 36725b7e6bbSNicholas Piggin } 36825b7e6bbSNicholas Piggin 369e4bb64c7SNicholas Piggin /* 370e4bb64c7SNicholas Piggin * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each 371e4bb64c7SNicholas Piggin * function definition. The reason for this is the noinstr section is placed 372e4bb64c7SNicholas Piggin * after the main text section, i.e., very far away from the interrupt entry 373e4bb64c7SNicholas Piggin * asm. That creates problems with fitting linker stubs when building large 374e4bb64c7SNicholas Piggin * kernels. 375e4bb64c7SNicholas Piggin */ 376e4bb64c7SNicholas Piggin #define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address 377e4bb64c7SNicholas Piggin 3788d41fc61SNicholas Piggin /** 3798d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function 3808d41fc61SNicholas Piggin * @func: Function name of the entry point 3818d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 3828d41fc61SNicholas Piggin */ 3838d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RAW(func) \ 3848d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 3858d41fc61SNicholas Piggin 3868d41fc61SNicholas Piggin /** 3878d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function 3888d41fc61SNicholas Piggin * @func: Function name of the entry point 3898d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 3908d41fc61SNicholas Piggin * 3918d41fc61SNicholas Piggin * @func is called from ASM entry code. 3928d41fc61SNicholas Piggin * 3938d41fc61SNicholas Piggin * This is a plain function which does no tracing, reconciling, etc. 3948d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 3958d41fc61SNicholas Piggin * body with a pair of curly brackets. 3968d41fc61SNicholas Piggin * 3978d41fc61SNicholas Piggin * raw interrupt handlers must not enable or disable interrupts, or 3988d41fc61SNicholas Piggin * schedule, tracing and instrumentation (ftrace, lockdep, etc) would 3998d41fc61SNicholas Piggin * not be advisable either, although may be possible in a pinch, the 4008d41fc61SNicholas Piggin * trace will look odd at least. 4018d41fc61SNicholas Piggin * 4028d41fc61SNicholas Piggin * A raw handler may call one of the other interrupt handler functions 4038d41fc61SNicholas Piggin * to be converted into that interrupt context without these restrictions. 4048d41fc61SNicholas Piggin * 4058d41fc61SNicholas Piggin * On PPC64, _RAW handlers may return with fast_interrupt_return. 4068d41fc61SNicholas Piggin * 4078d41fc61SNicholas Piggin * Specific handlers may have additional restrictions. 4088d41fc61SNicholas Piggin */ 4098d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RAW(func) \ 4108d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 4118d41fc61SNicholas Piggin \ 412e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 4138d41fc61SNicholas Piggin { \ 4148d41fc61SNicholas Piggin long ret; \ 4158d41fc61SNicholas Piggin \ 416ff0b0d6eSNicholas Piggin __hard_RI_enable(); \ 417ff0b0d6eSNicholas Piggin \ 4188d41fc61SNicholas Piggin ret = ____##func (regs); \ 4198d41fc61SNicholas Piggin \ 4208d41fc61SNicholas Piggin return ret; \ 4218d41fc61SNicholas Piggin } \ 422e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4238d41fc61SNicholas Piggin \ 4248d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 4258d41fc61SNicholas Piggin 4268d41fc61SNicholas Piggin /** 4278d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function 4288d41fc61SNicholas Piggin * @func: Function name of the entry point 4298d41fc61SNicholas Piggin */ 4308d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER(func) \ 4318d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs) 4328d41fc61SNicholas Piggin 4338d41fc61SNicholas Piggin /** 4348d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function 4358d41fc61SNicholas Piggin * @func: Function name of the entry point 4368d41fc61SNicholas Piggin * 4378d41fc61SNicholas Piggin * @func is called from ASM entry code. 4388d41fc61SNicholas Piggin * 4398d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 4408d41fc61SNicholas Piggin * body with a pair of curly brackets. 4418d41fc61SNicholas Piggin */ 4428d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER(func) \ 4438d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \ 4448d41fc61SNicholas Piggin \ 445e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \ 4468d41fc61SNicholas Piggin { \ 447973e2e64SChristophe Leroy interrupt_enter_prepare(regs); \ 44825b7e6bbSNicholas Piggin \ 4498d41fc61SNicholas Piggin ____##func (regs); \ 45025b7e6bbSNicholas Piggin \ 451973e2e64SChristophe Leroy interrupt_exit_prepare(regs); \ 4528d41fc61SNicholas Piggin } \ 453e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4548d41fc61SNicholas Piggin \ 4558d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs) 4568d41fc61SNicholas Piggin 4578d41fc61SNicholas Piggin /** 4588d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function 4598d41fc61SNicholas Piggin * @func: Function name of the entry point 4608d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 4618d41fc61SNicholas Piggin */ 4628d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_RET(func) \ 4638d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 4648d41fc61SNicholas Piggin 4658d41fc61SNicholas Piggin /** 4668d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function 4678d41fc61SNicholas Piggin * @func: Function name of the entry point 4688d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 4698d41fc61SNicholas Piggin * 4708d41fc61SNicholas Piggin * @func is called from ASM entry code. 4718d41fc61SNicholas Piggin * 4728d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 4738d41fc61SNicholas Piggin * body with a pair of curly brackets. 4748d41fc61SNicholas Piggin */ 4758d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_RET(func) \ 4768d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 4778d41fc61SNicholas Piggin \ 478e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 4798d41fc61SNicholas Piggin { \ 4808d41fc61SNicholas Piggin long ret; \ 4818d41fc61SNicholas Piggin \ 482973e2e64SChristophe Leroy interrupt_enter_prepare(regs); \ 48325b7e6bbSNicholas Piggin \ 4848d41fc61SNicholas Piggin ret = ____##func (regs); \ 4858d41fc61SNicholas Piggin \ 486973e2e64SChristophe Leroy interrupt_exit_prepare(regs); \ 48725b7e6bbSNicholas Piggin \ 4888d41fc61SNicholas Piggin return ret; \ 4898d41fc61SNicholas Piggin } \ 490e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 4918d41fc61SNicholas Piggin \ 4928d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 4938d41fc61SNicholas Piggin 4948d41fc61SNicholas Piggin /** 4958d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function 4968d41fc61SNicholas Piggin * @func: Function name of the entry point 4978d41fc61SNicholas Piggin */ 4988d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ 4998d41fc61SNicholas Piggin __visible void func(struct pt_regs *regs) 5008d41fc61SNicholas Piggin 5018d41fc61SNicholas Piggin /** 5028d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function 5038d41fc61SNicholas Piggin * @func: Function name of the entry point 5048d41fc61SNicholas Piggin * 5058d41fc61SNicholas Piggin * @func is called from ASM entry code. 5068d41fc61SNicholas Piggin * 5078d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 5088d41fc61SNicholas Piggin * body with a pair of curly brackets. 5098d41fc61SNicholas Piggin */ 5108d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ 5118d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs); \ 5128d41fc61SNicholas Piggin \ 513e4bb64c7SNicholas Piggin interrupt_handler void func(struct pt_regs *regs) \ 5148d41fc61SNicholas Piggin { \ 515973e2e64SChristophe Leroy interrupt_async_enter_prepare(regs); \ 51625b7e6bbSNicholas Piggin \ 5178d41fc61SNicholas Piggin ____##func (regs); \ 51825b7e6bbSNicholas Piggin \ 519973e2e64SChristophe Leroy interrupt_async_exit_prepare(regs); \ 5208d41fc61SNicholas Piggin } \ 521e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 5228d41fc61SNicholas Piggin \ 5238d41fc61SNicholas Piggin static __always_inline void ____##func(struct pt_regs *regs) 5248d41fc61SNicholas Piggin 5258d41fc61SNicholas Piggin /** 5268d41fc61SNicholas Piggin * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function 5278d41fc61SNicholas Piggin * @func: Function name of the entry point 5288d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 5298d41fc61SNicholas Piggin */ 5308d41fc61SNicholas Piggin #define DECLARE_INTERRUPT_HANDLER_NMI(func) \ 5318d41fc61SNicholas Piggin __visible long func(struct pt_regs *regs) 5328d41fc61SNicholas Piggin 5338d41fc61SNicholas Piggin /** 5348d41fc61SNicholas Piggin * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function 5358d41fc61SNicholas Piggin * @func: Function name of the entry point 5368d41fc61SNicholas Piggin * @returns: Returns a value back to asm caller 5378d41fc61SNicholas Piggin * 5388d41fc61SNicholas Piggin * @func is called from ASM entry code. 5398d41fc61SNicholas Piggin * 5408d41fc61SNicholas Piggin * The macro is written so it acts as function definition. Append the 5418d41fc61SNicholas Piggin * body with a pair of curly brackets. 5428d41fc61SNicholas Piggin */ 5438d41fc61SNicholas Piggin #define DEFINE_INTERRUPT_HANDLER_NMI(func) \ 5448d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs); \ 5458d41fc61SNicholas Piggin \ 546e4bb64c7SNicholas Piggin interrupt_handler long func(struct pt_regs *regs) \ 5478d41fc61SNicholas Piggin { \ 54825b7e6bbSNicholas Piggin struct interrupt_nmi_state state; \ 5498d41fc61SNicholas Piggin long ret; \ 5508d41fc61SNicholas Piggin \ 55125b7e6bbSNicholas Piggin interrupt_nmi_enter_prepare(regs, &state); \ 55225b7e6bbSNicholas Piggin \ 5538d41fc61SNicholas Piggin ret = ____##func (regs); \ 5548d41fc61SNicholas Piggin \ 55525b7e6bbSNicholas Piggin interrupt_nmi_exit_prepare(regs, &state); \ 55625b7e6bbSNicholas Piggin \ 5578d41fc61SNicholas Piggin return ret; \ 5588d41fc61SNicholas Piggin } \ 559e4bb64c7SNicholas Piggin NOKPROBE_SYMBOL(func); \ 5608d41fc61SNicholas Piggin \ 5618d41fc61SNicholas Piggin static __always_inline long ____##func(struct pt_regs *regs) 5628d41fc61SNicholas Piggin 5633a96570fSNicholas Piggin 5643a96570fSNicholas Piggin /* Interrupt handlers */ 5653a96570fSNicholas Piggin /* kernel/traps.c */ 5663a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); 5673a96570fSNicholas Piggin #ifdef CONFIG_PPC_BOOK3S_64 568f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception_async); 5693a96570fSNicholas Piggin #endif 570f08fb25bSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); 5713a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SMIException); 5723a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); 5733a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(unknown_exception); 5743a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); 5753db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(unknown_nmi_exception); 5763a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); 5773a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(RunModeException); 5783a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(single_step_exception); 5793a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(program_check_exception); 5803a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); 5813a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(alignment_exception); 5823a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(StackOverflow); 5833a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); 5843a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); 5853a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); 5863a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); 5873a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); 5883a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); 5893a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); 5903a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); 5913a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); 5923a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); 5933a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); 5943a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(DebugException); 5953a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); 5963a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(CacheLockingException); 5973a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); 5983a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); 5993db8aa10SNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(WatchdogException); 6003a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); 6013a96570fSNicholas Piggin 6023a96570fSNicholas Piggin /* slb.c */ 6033a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); 604935b534cSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_segment_interrupt); 6053a96570fSNicholas Piggin 6063a96570fSNicholas Piggin /* hash_utils.c */ 6078b91cee5SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_hash_fault); 6083a96570fSNicholas Piggin 6093a96570fSNicholas Piggin /* fault.c */ 610c45ba4f4SNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_page_fault); 6113a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); 6123a96570fSNicholas Piggin 6133a96570fSNicholas Piggin /* process.c */ 6143a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER(do_break); 6153a96570fSNicholas Piggin 6163a96570fSNicholas Piggin /* time.c */ 6173a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); 6183a96570fSNicholas Piggin 6193a96570fSNicholas Piggin /* mce.c */ 6203a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); 6213a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); 6223a96570fSNicholas Piggin 6233a96570fSNicholas Piggin DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); 6243a96570fSNicholas Piggin 62598694166SChristophe Leroy /* irq.c */ 62698694166SChristophe Leroy DECLARE_INTERRUPT_HANDLER_ASYNC(do_IRQ); 62798694166SChristophe Leroy 628a58cbed6SChristophe Leroy void __noreturn unrecoverable_exception(struct pt_regs *regs); 6290b736881SChristophe Leroy 6303a96570fSNicholas Piggin void replay_system_reset(void); 6313a96570fSNicholas Piggin void replay_soft_interrupts(void); 6323a96570fSNicholas Piggin 633e6f8a6c8SNicholas Piggin static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) 634e6f8a6c8SNicholas Piggin { 635e6f8a6c8SNicholas Piggin if (!arch_irq_disabled_regs(regs)) 636e6f8a6c8SNicholas Piggin local_irq_enable(); 637e6f8a6c8SNicholas Piggin } 638e6f8a6c8SNicholas Piggin 639*76222808SChristophe Leroy long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, 640*76222808SChristophe Leroy unsigned long r0, struct pt_regs *regs); 641*76222808SChristophe Leroy notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv); 642*76222808SChristophe Leroy notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs); 643*76222808SChristophe Leroy notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs); 644*76222808SChristophe Leroy #ifdef CONFIG_PPC64 645*76222808SChristophe Leroy unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs); 646*76222808SChristophe Leroy unsigned long interrupt_exit_user_restart(struct pt_regs *regs); 647*76222808SChristophe Leroy unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs); 648*76222808SChristophe Leroy #endif 649*76222808SChristophe Leroy 6500f5eb28aSChristophe Leroy #endif /* __ASSEMBLY__ */ 6510f5eb28aSChristophe Leroy 6528d41fc61SNicholas Piggin #endif /* _ASM_POWERPC_INTERRUPT_H */ 653