1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * IRQ subsystem internal functions and variables: 4 * 5 * Do not ever include this file from anything else than 6 * kernel/irq/. Do not even think about using any information outside 7 * of this file for your non core code. 8 */ 9 #include <linux/irqdesc.h> 10 #include <linux/kernel_stat.h> 11 #include <linux/pm_runtime.h> 12 #include <linux/rcuref.h> 13 #include <linux/sched/clock.h> 14 15 #include "debugfs.h" 16 #include "proc.h" 17 18 #ifdef CONFIG_SPARSE_IRQ 19 # define MAX_SPARSE_IRQS INT_MAX 20 #else 21 # define MAX_SPARSE_IRQS NR_IRQS 22 #endif 23 24 #define istate core_internal_state__do_not_mess_with_it 25 26 extern bool noirqdebug; 27 extern int irq_poll_cpu; 28 extern unsigned int total_nr_irqs; 29 30 extern struct irqaction chained_action; 31 32 /* 33 * Bits used by threaded handlers: 34 * IRQTF_RUNTHREAD - signals that the interrupt handler thread should run 35 * IRQTF_WARNED - warning "IRQ_WAKE_THREAD w/o thread_fn" has been printed 36 * IRQTF_AFFINITY - irq thread is requested to adjust affinity 37 * IRQTF_FORCED_THREAD - irq action is force threaded 38 * IRQTF_READY - signals that irq thread is ready 39 */ 40 enum { 41 IRQTF_RUNTHREAD, 42 IRQTF_WARNED, 43 IRQTF_AFFINITY, 44 IRQTF_FORCED_THREAD, 45 IRQTF_READY, 46 }; 47 48 /* 49 * Bit masks for desc->core_internal_state__do_not_mess_with_it 50 * 51 * IRQS_AUTODETECT - autodetection in progress 52 * IRQS_SPURIOUS_DISABLED - was disabled due to spurious interrupt 53 * detection 54 * IRQS_POLL_INPROGRESS - polling in progress 55 * IRQS_ONESHOT - irq is not unmasked in primary handler 56 * IRQS_REPLAY - irq has been resent and will not be resent 57 * again until the handler has run and cleared 58 * this flag. 59 * IRQS_WAITING - irq is waiting 60 * IRQS_PENDING - irq needs to be resent and should be resent 61 * at the next available opportunity. 62 * IRQS_SUSPENDED - irq is suspended 63 * IRQS_NMI - irq line is used to deliver NMIs 64 * IRQS_SYSFS - descriptor has been added to sysfs 65 */ 66 enum { 67 IRQS_AUTODETECT = 0x00000001, 68 IRQS_SPURIOUS_DISABLED = 0x00000002, 69 IRQS_POLL_INPROGRESS = 0x00000008, 70 IRQS_ONESHOT = 0x00000020, 71 IRQS_REPLAY = 0x00000040, 72 IRQS_WAITING = 0x00000080, 73 IRQS_PENDING = 0x00000200, 74 IRQS_SUSPENDED = 0x00000800, 75 IRQS_TIMINGS = 0x00001000, 76 IRQS_NMI = 0x00002000, 77 IRQS_SYSFS = 0x00004000, 78 }; 79 80 #include "debug.h" 81 #include "settings.h" 82 83 extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags); 84 extern void __disable_irq(struct irq_desc *desc); 85 extern void __enable_irq(struct irq_desc *desc); 86 87 #define IRQ_RESEND true 88 #define IRQ_NORESEND false 89 90 #define IRQ_START_FORCE true 91 #define IRQ_START_COND false 92 93 extern int irq_activate(struct irq_desc *desc); 94 extern int irq_activate_and_startup(struct irq_desc *desc, bool resend); 95 extern int irq_startup(struct irq_desc *desc, bool resend, bool force); 96 extern void irq_startup_managed(struct irq_desc *desc); 97 98 extern void irq_shutdown(struct irq_desc *desc); 99 extern void irq_shutdown_and_deactivate(struct irq_desc *desc); 100 extern void irq_disable(struct irq_desc *desc); 101 extern void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu); 102 extern void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu); 103 extern void mask_irq(struct irq_desc *desc); 104 extern void unmask_irq(struct irq_desc *desc); 105 extern void unmask_threaded_irq(struct irq_desc *desc); 106 107 #ifdef CONFIG_SPARSE_IRQ 108 static __always_inline void irq_mark_irq(unsigned int irq) { } 109 void irq_desc_free_rcu(struct irq_desc *desc); 110 111 static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) 112 { 113 return rcuref_get(&desc->refcnt); 114 } 115 116 static __always_inline void irq_desc_put_ref(struct irq_desc *desc) 117 { 118 if (rcuref_put(&desc->refcnt)) 119 irq_desc_free_rcu(desc); 120 } 121 #else 122 extern void irq_mark_irq(unsigned int irq); 123 static __always_inline bool irq_desc_get_ref(struct irq_desc *desc) { return true; } 124 static __always_inline void irq_desc_put_ref(struct irq_desc *desc) { } 125 #endif 126 127 irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc); 128 irqreturn_t handle_irq_event_percpu(struct irq_desc *desc); 129 irqreturn_t handle_irq_event(struct irq_desc *desc); 130 131 /* Resending of interrupts :*/ 132 int check_irq_resend(struct irq_desc *desc, bool inject); 133 void clear_irq_resend(struct irq_desc *desc); 134 void irq_resend_init(struct irq_desc *desc); 135 void __irq_wake_thread(struct irq_desc *desc, struct irqaction *action); 136 137 void wake_threads_waitq(struct irq_desc *desc); 138 139 #ifdef CONFIG_PROC_FS 140 extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); 141 extern void unregister_irq_proc(unsigned int irq, struct irq_desc *desc); 142 extern void register_handler_proc(unsigned int irq, struct irqaction *action); 143 extern void unregister_handler_proc(unsigned int irq, struct irqaction *action); 144 void irq_proc_update_valid(struct irq_desc *desc); 145 #else 146 static inline void register_irq_proc(unsigned int irq, struct irq_desc *desc) { } 147 static inline void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) { } 148 static inline void register_handler_proc(unsigned int irq, 149 struct irqaction *action) { } 150 static inline void unregister_handler_proc(unsigned int irq, 151 struct irqaction *action) { } 152 static inline void irq_proc_update_valid(struct irq_desc *desc) { } 153 #endif 154 155 struct irq_desc *irq_find_desc_at_or_after(unsigned int offset); 156 157 extern bool irq_can_set_affinity_usr(unsigned int irq); 158 159 extern int irq_do_set_affinity(struct irq_data *data, 160 const struct cpumask *dest, bool force); 161 extern void irq_affinity_schedule_notify_work(struct irq_desc *desc); 162 163 #ifdef CONFIG_SMP 164 extern int irq_setup_affinity(struct irq_desc *desc); 165 #else 166 static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; } 167 #endif 168 169 #define for_each_action_of_desc(desc, act) \ 170 for (act = desc->action; act; act = act->next) 171 172 /* Inline functions for support of irq chips on slow busses */ 173 static inline void chip_bus_lock(struct irq_desc *desc) 174 { 175 if (unlikely(desc->irq_data.chip->irq_bus_lock)) 176 desc->irq_data.chip->irq_bus_lock(&desc->irq_data); 177 } 178 179 static inline void chip_bus_sync_unlock(struct irq_desc *desc) 180 { 181 if (unlikely(desc->irq_data.chip->irq_bus_sync_unlock)) 182 desc->irq_data.chip->irq_bus_sync_unlock(&desc->irq_data); 183 } 184 185 #define _IRQ_DESC_CHECK (1 << 0) 186 #define _IRQ_DESC_PERCPU (1 << 1) 187 188 #define IRQ_GET_DESC_CHECK_GLOBAL (_IRQ_DESC_CHECK) 189 #define IRQ_GET_DESC_CHECK_PERCPU (_IRQ_DESC_CHECK | _IRQ_DESC_PERCPU) 190 191 struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus, 192 unsigned int check); 193 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus); 194 195 __DEFINE_CLASS_IS_CONDITIONAL(irqdesc_lock, true); 196 __DEFINE_UNLOCK_GUARD(irqdesc_lock, struct irq_desc, 197 if (_T->lock) __irq_put_desc_unlock(_T->lock, _T->flags, _T->bus), 198 unsigned long flags; bool bus); 199 200 static inline class_irqdesc_lock_t class_irqdesc_lock_constructor(unsigned int irq, bool bus, 201 unsigned int check) 202 { 203 class_irqdesc_lock_t _t = { .bus = bus, }; 204 205 _t.lock = __irq_get_desc_lock(irq, &_t.flags, bus, check); 206 207 return _t; 208 } 209 210 #define scoped_irqdesc_get_and_lock(_irq, _check) \ 211 scoped_guard(irqdesc_lock, _irq, false, _check) 212 213 #define scoped_irqdesc_get_and_buslock(_irq, _check) \ 214 scoped_guard(irqdesc_lock, _irq, true, _check) 215 216 #define scoped_irqdesc ((struct irq_desc *)(__guard_ptr(irqdesc_lock)(&scope))) 217 218 #define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) 219 220 static inline unsigned int irqd_get(struct irq_data *d) 221 { 222 return __irqd_to_state(d); 223 } 224 225 /* 226 * Manipulation functions for irq_data.state 227 */ 228 static inline void irqd_set_move_pending(struct irq_data *d) 229 { 230 __irqd_to_state(d) |= IRQD_SETAFFINITY_PENDING; 231 } 232 233 static inline void irqd_clr_move_pending(struct irq_data *d) 234 { 235 __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING; 236 } 237 238 static inline void irqd_set_managed_shutdown(struct irq_data *d) 239 { 240 __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN; 241 } 242 243 static inline void irqd_clr_managed_shutdown(struct irq_data *d) 244 { 245 __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN; 246 } 247 248 static inline void irqd_clear(struct irq_data *d, unsigned int mask) 249 { 250 __irqd_to_state(d) &= ~mask; 251 } 252 253 static inline void irqd_set(struct irq_data *d, unsigned int mask) 254 { 255 __irqd_to_state(d) |= mask; 256 } 257 258 static inline bool irqd_has_set(struct irq_data *d, unsigned int mask) 259 { 260 return __irqd_to_state(d) & mask; 261 } 262 263 static inline void irq_state_set_disabled(struct irq_desc *desc) 264 { 265 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); 266 } 267 268 static inline void irq_state_set_masked(struct irq_desc *desc) 269 { 270 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); 271 } 272 273 #undef __irqd_to_state 274 275 static inline void __kstat_incr_irqs_this_cpu(struct irq_desc *desc) 276 { 277 __this_cpu_inc(desc->kstat_irqs->cnt); 278 __this_cpu_inc(kstat.irqs_sum); 279 } 280 281 static inline void kstat_incr_irqs_this_cpu(struct irq_desc *desc) 282 { 283 __kstat_incr_irqs_this_cpu(desc); 284 desc->tot_count++; 285 } 286 287 static inline int irq_desc_get_node(struct irq_desc *desc) 288 { 289 return irq_common_data_get_node(&desc->irq_common_data); 290 } 291 292 static inline int irq_desc_is_chained(struct irq_desc *desc) 293 { 294 return (desc->action && desc->action == &chained_action); 295 } 296 297 static inline bool irq_is_nmi(struct irq_desc *desc) 298 { 299 return desc->istate & IRQS_NMI; 300 } 301 302 #ifdef CONFIG_PM_SLEEP 303 void irq_pm_handle_wakeup(struct irq_desc *desc); 304 void irq_pm_install_action(struct irq_desc *desc, struct irqaction *action); 305 void irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action); 306 #else 307 static inline void irq_pm_handle_wakeup(struct irq_desc *desc) { } 308 static inline void 309 irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { } 310 static inline void 311 irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { } 312 #endif 313 314 #ifdef CONFIG_GENERIC_IRQ_CHIP 315 void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, 316 int num_ct, unsigned int irq_base, 317 void __iomem *reg_base, irq_flow_handler_t handler); 318 #else 319 static inline void 320 irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, 321 int num_ct, unsigned int irq_base, 322 void __iomem *reg_base, irq_flow_handler_t handler) { } 323 #endif /* CONFIG_GENERIC_IRQ_CHIP */ 324 325 #ifdef CONFIG_GENERIC_PENDING_IRQ 326 static inline bool irq_can_move_pcntxt(struct irq_data *data) 327 { 328 return !(data->chip->flags & IRQCHIP_MOVE_DEFERRED); 329 } 330 static inline bool irq_move_pending(struct irq_data *data) 331 { 332 return irqd_is_setaffinity_pending(data); 333 } 334 static inline void 335 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) 336 { 337 cpumask_copy(desc->pending_mask, mask); 338 } 339 static inline void 340 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) 341 { 342 cpumask_copy(mask, desc->pending_mask); 343 } 344 static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc) 345 { 346 return desc->pending_mask; 347 } 348 bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear); 349 void irq_force_complete_move(struct irq_desc *desc); 350 #else /* CONFIG_GENERIC_PENDING_IRQ */ 351 static inline bool irq_can_move_pcntxt(struct irq_data *data) 352 { 353 return true; 354 } 355 static inline bool irq_move_pending(struct irq_data *data) 356 { 357 return false; 358 } 359 static inline void 360 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) 361 { 362 } 363 static inline void 364 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) 365 { 366 } 367 static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc) 368 { 369 return NULL; 370 } 371 static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear) 372 { 373 return false; 374 } 375 static inline void irq_force_complete_move(struct irq_desc *desc) { } 376 #endif /* !CONFIG_GENERIC_PENDING_IRQ */ 377 378 #if !defined(CONFIG_IRQ_DOMAIN) || !defined(CONFIG_IRQ_DOMAIN_HIERARCHY) 379 static inline int irq_domain_activate_irq(struct irq_data *data, bool reserve) 380 { 381 irqd_set_activated(data); 382 return 0; 383 } 384 static inline void irq_domain_deactivate_irq(struct irq_data *data) 385 { 386 irqd_clr_activated(data); 387 } 388 #endif 389 390 static inline struct irq_data *irqd_get_parent_data(struct irq_data *irqd) 391 { 392 #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY 393 return irqd->parent_data; 394 #else 395 return NULL; 396 #endif 397 } 398