1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __LINUX_INTERRUPT_RC_H 3 #define __LINUX_INTERRUPT_RC_H 4 5 /* 6 * include/linux/interrupt_rc.h - refcounted local processor interrupt 7 * management. 8 * 9 * Since the implementation of this API currently depends on 10 * local_irq_save()/local_irq_restore(), we split this into its own header to 11 * make it easier to include without hitting circular header dependencies. 12 */ 13 14 #include <linux/debug_locks.h> 15 #include <linux/irqflags.h> 16 #include <linux/preempt.h> 17 #include <linux/processor.h> 18 #include <linux/smp.h> 19 20 #ifndef MODULE 21 /* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ 22 DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); 23 __local_interrupt_save_state(unsigned long flags)24static __always_inline void __local_interrupt_save_state(unsigned long flags) 25 { 26 raw_cpu_write(local_interrupt_disable_state, flags); 27 } 28 __local_interrupt_enable(void)29static __always_inline void __local_interrupt_enable(void) 30 { 31 unsigned long flags = raw_cpu_read(local_interrupt_disable_state); 32 33 local_irq_restore(flags); 34 } 35 36 #ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE _local_interrupt_save_state(unsigned long flags)37static __always_inline void _local_interrupt_save_state(unsigned long flags) 38 { 39 __local_interrupt_save_state(flags); 40 } 41 _local_interrupt_enable(void)42static __always_inline void _local_interrupt_enable(void) 43 { 44 __local_interrupt_enable(); 45 } 46 #else 47 extern void _local_interrupt_save_state(unsigned long flags); 48 extern void _local_interrupt_enable(void); 49 #endif 50 51 #else /* !MODULE */ 52 extern void _local_interrupt_save_state(unsigned long flags); 53 extern void _local_interrupt_enable(void); 54 #endif /* !MODULE */ 55 56 #define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) 57 #define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) 58 local_interrupt_disable(void)59static inline void local_interrupt_disable(void) 60 { 61 int new_count; 62 unsigned long flags; 63 64 WARN_ON_ONCE(in_nmi()); 65 66 local_irq_save(flags); 67 new_count = hardirq_disable_enter(); 68 69 /* Is hardirq disable count overflow soon? */ 70 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && 71 DEBUG_LOCKS_WARN_ON(((new_count & HARDIRQ_DISABLE_MASK) + 72 (10 << HARDIRQ_DISABLE_SHIFT)) >= 73 HARDIRQ_DISABLE_MASK)) 74 return; 75 76 if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) 77 _local_interrupt_save_state(flags); 78 } 79 local_interrupt_enable(void)80static inline void local_interrupt_enable(void) 81 { 82 int new_count; 83 84 /* Unpaired local_interrupt_enable()? Warn and abort. */ 85 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && 86 DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0)) 87 return; 88 89 new_count = hardirq_disable_exit(); 90 91 if ((new_count & HARDIRQ_DISABLE_MASK) == 0) 92 _local_interrupt_enable(); 93 } 94 95 #endif /* !__LINUX_INTERRUPT_RC_H */ 96