1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
7  * information is available in Documentation/core-api/genericirq.rst
10 #include <linux/irq.h>
23  * lockdep: we want to handle all irq_desc locks as a single lock-class:
57 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,  in alloc_masks()
59 		return -ENOMEM;  in alloc_masks()
62 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,  in alloc_masks()
64 		free_cpumask_var(desc->irq_common_data.affinity);  in alloc_masks()
65 		return -ENOMEM;  in alloc_masks()
70 	if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {  in alloc_masks()
72 		free_cpumask_var(desc->irq_common_data.effective_affinity);  in alloc_masks()
74 		free_cpumask_var(desc->irq_common_data.affinity);  in alloc_masks()
75 		return -ENOMEM;  in alloc_masks()
86 	cpumask_copy(desc->irq_common_data.affinity, affinity);  in desc_smp_init()
89 	cpumask_clear(desc->pending_mask);  in desc_smp_init()
92 	desc->irq_common_data.node = node;  in desc_smp_init()
99 	free_cpumask_var(desc->pending_mask);  in free_masks()
101 	free_cpumask_var(desc->irq_common_data.affinity);  in free_masks()
103 	free_cpumask_var(desc->irq_common_data.effective_affinity);  in free_masks()
115 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,  in desc_set_defaults()  argument
120 	desc->irq_common_data.handler_data = NULL;  in desc_set_defaults()
121 	desc->irq_common_data.msi_desc = NULL;  in desc_set_defaults()
123 	desc->irq_data.common = &desc->irq_common_data;  in desc_set_defaults()
124 	desc->irq_data.irq = irq;  in desc_set_defaults()
125 	desc->irq_data.chip = &no_irq_chip;  in desc_set_defaults()
126 	desc->irq_data.chip_data = NULL;  in desc_set_defaults()
128 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);  in desc_set_defaults()
129 	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);  in desc_set_defaults()
130 	desc->handle_irq = handle_bad_irq;  in desc_set_defaults()
131 	desc->depth = 1;  in desc_set_defaults()
132 	desc->irq_count = 0;  in desc_set_defaults()
133 	desc->irqs_unhandled = 0;  in desc_set_defaults()
134 	desc->tot_count = 0;  in desc_set_defaults()
135 	desc->name = NULL;  in desc_set_defaults()
136 	desc->owner = owner;  in desc_set_defaults()
138 		*per_cpu_ptr(desc->kstat_irqs, cpu) = (struct irqstat) { };  in desc_set_defaults()
145  * irq_get_nr_irqs() - Number of interrupts supported by the system.
154  * irq_set_nr_irqs() - Set the number of interrupts supported by the system.
179 		return -ENOSPC;  in irq_find_free_area()
194 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)  in irq_insert_desc()  argument
196 	MA_STATE(mas, &sparse_irqs, irq, irq);  in irq_insert_desc()
200 static void delete_irq_desc(unsigned int irq)  in delete_irq_desc()  argument
202 	MA_STATE(mas, &sparse_irqs, irq, irq);  in delete_irq_desc()
210 static int init_desc(struct irq_desc *desc, int irq, int node,  in init_desc()  argument
215 	desc->kstat_irqs = alloc_percpu(struct irqstat);  in init_desc()
216 	if (!desc->kstat_irqs)  in init_desc()
217 		return -ENOMEM;  in init_desc()
220 		free_percpu(desc->kstat_irqs);  in init_desc()
221 		return -ENOMEM;  in init_desc()
224 	raw_spin_lock_init(&desc->lock);  in init_desc()
225 	lockdep_set_class(&desc->lock, &irq_desc_lock_class);  in init_desc()
226 	mutex_init(&desc->request_mutex);  in init_desc()
227 	init_waitqueue_head(&desc->wait_for_threads);  in init_desc()
228 	desc_set_defaults(irq, desc, node, affinity, owner);  in init_desc()
229 	irqd_set(&desc->irq_data, flags);  in init_desc()
232 	kobject_init(&desc->kobj, &irq_kobj_type);  in init_desc()
233 	init_rcu_head(&desc->rcu);  in init_desc()
272 	guard(raw_spinlock_irq)(&desc->lock);  in chip_name_show()
273 	if (desc->irq_data.chip && desc->irq_data.chip->name)  in chip_name_show()
274 		return sysfs_emit(buf, "%s\n", desc->irq_data.chip->name);  in chip_name_show()
283 	guard(raw_spinlock_irq)(&desc->lock);  in hwirq_show()
284 	if (desc->irq_data.domain)  in hwirq_show()
285 		return sysfs_emit(buf, "%lu\n", desc->irq_data.hwirq);  in hwirq_show()
294 	guard(raw_spinlock_irq)(&desc->lock);  in type_show()
295 	return sysfs_emit(buf, "%s\n", irqd_is_level_type(&desc->irq_data) ? "level" : "edge");  in type_show()
304 	guard(raw_spinlock_irq)(&desc->lock);  in wakeup_show()
305 	return sysfs_emit(buf, "%s\n", str_enabled_disabled(irqd_is_wakeup_set(&desc->irq_data)));  in wakeup_show()
313 	guard(raw_spinlock_irq)(&desc->lock);  in name_show()
314 	if (desc->name)  in name_show()
315 		return sysfs_emit(buf, "%s\n", desc->name);  in name_show()
327 	scoped_guard(raw_spinlock_irq, &desc->lock) {  in actions_show()
329 			ret += sysfs_emit_at(buf, ret, "%s%s", p, action->name);  in actions_show()
350 ATTRIBUTE_GROUPS(irq);
358 static void irq_sysfs_add(int irq, struct irq_desc *desc)  in irq_sysfs_add()  argument
366 		if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))  in irq_sysfs_add()
367 			pr_warn("Failed to add kobject for irq %d\n", irq);  in irq_sysfs_add()
369 			desc->istate |= IRQS_SYSFS;  in irq_sysfs_add()
381 	if (desc->istate & IRQS_SYSFS)  in irq_sysfs_del()
382 		kobject_del(&desc->kobj);  in irq_sysfs_del()
388 	int irq;  in irq_sysfs_init()  local
390 	/* Prevent concurrent irq alloc/free */  in irq_sysfs_init()
392 	irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);  in irq_sysfs_init()
394 		return -ENOMEM;  in irq_sysfs_init()
397 	for_each_irq_desc(irq, desc)  in irq_sysfs_init()
398 		irq_sysfs_add(irq, desc);  in irq_sysfs_init()
409 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}  in irq_sysfs_add()  argument
414 struct irq_desc *irq_to_desc(unsigned int irq)  in irq_to_desc()  argument
416 	return mtree_load(&sparse_irqs, irq);  in irq_to_desc()
432 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,  in alloc_desc()  argument
443 	ret = init_desc(desc, irq, node, flags, affinity, owner);  in alloc_desc()
457 	free_percpu(desc->kstat_irqs);  in irq_kobj_release()
465 	kobject_put(&desc->kobj);  in delayed_free_desc()
468 static void free_desc(unsigned int irq)  in free_desc()  argument
470 	struct irq_desc *desc = irq_to_desc(irq);  in free_desc()
473 	unregister_irq_proc(irq, desc);  in free_desc()
485 	delete_irq_desc(irq);  in free_desc()
493 	call_rcu(&desc->rcu, delayed_free_desc);  in free_desc()
496 static int alloc_descs(unsigned int start, unsigned int cnt, int node,  in alloc_descs()  argument
507 				return -EINVAL;  in alloc_descs()
516 			if (affinity->is_managed) {  in alloc_descs()
521 			mask = &affinity->mask;  in alloc_descs()
526 		desc = alloc_desc(start + i, node, flags, mask, owner);  in alloc_descs()
529 		irq_insert_desc(start + i, desc);  in alloc_descs()
530 		irq_sysfs_add(start + i, desc);  in alloc_descs()
531 		irq_add_debugfs_entry(start + i, desc);  in alloc_descs()
533 	return start;  in alloc_descs()
536 	for (i--; i >= 0; i--)  in alloc_descs()
537 		free_desc(start + i);  in alloc_descs()
538 	return -ENOMEM;  in alloc_descs()
580 	[0 ... NR_IRQS-1] = {
583 		.lock		= __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
607 	while (--i >= 0) {  in early_irq_init()
615 struct irq_desc *irq_to_desc(unsigned int irq)  in irq_to_desc()  argument
617 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;  in irq_to_desc()
621 static void free_desc(unsigned int irq)  in free_desc()  argument
623 	struct irq_desc *desc = irq_to_desc(irq);  in free_desc()
625 	scoped_guard(raw_spinlock_irqsave, &desc->lock)  in free_desc()
626 		desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);  in free_desc()
627 	delete_irq_desc(irq);  in free_desc()
630 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,  in alloc_descs()  argument
637 		struct irq_desc *desc = irq_to_desc(start + i);  in alloc_descs()
639 		desc->owner = owner;  in alloc_descs()
640 		irq_insert_desc(start + i, desc);  in alloc_descs()
642 	return start;  in alloc_descs()
650 void irq_mark_irq(unsigned int irq)  in irq_mark_irq()  argument
653 	irq_insert_desc(irq, irq_desc + irq);  in irq_mark_irq()
657 void irq_init_desc(unsigned int irq)  in irq_init_desc()  argument
659 	free_desc(irq);  in irq_init_desc()
670 		return -EINVAL;  in handle_irq_desc()
674 		return -EPERM;  in handle_irq_desc()
681  * generic_handle_irq - Invoke the handler for a particular irq
682  * @irq:	The irq number to handle
684  * Returns:	0 on success, or -EINVAL if conversion has failed
686  * 		This function must be called from an IRQ context with irq regs
689 int generic_handle_irq(unsigned int irq)  in generic_handle_irq()  argument
691 	return handle_irq_desc(irq_to_desc(irq));  in generic_handle_irq()
696  * generic_handle_irq_safe - Invoke the handler for a particular irq from any
698  * @irq:	The irq number to handle
702  * This function can be called from any context (IRQ or process context). It
703  * will report an error if not invoked from IRQ context and the irq has been
704  * marked to enforce IRQ-context only.
706 int generic_handle_irq_safe(unsigned int irq)  in generic_handle_irq_safe()  argument
712 	ret = handle_irq_desc(irq_to_desc(irq));  in generic_handle_irq_safe()
720  * generic_handle_domain_irq - Invoke the handler for a HW irq belonging
723  * @hwirq:	The HW irq number to convert to a logical one
725  * Returns:	0 on success, or -EINVAL if conversion has failed
727  * 		This function must be called from an IRQ context with irq regs
737  * generic_handle_irq_safe - Invoke the handler for a HW irq belonging
740  * @hwirq:	The HW irq number to convert to a logical one
744  * This function can be called from any context (IRQ or process
745  * context). If the interrupt is marked as 'enforce IRQ-context only' then
761  * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging
764  * @hwirq:	The HW irq number to convert to a logical one
766  * Returns:	0 on success, or -EINVAL if conversion has failed
768  * 		This function must be called from an NMI context with irq regs
781  * irq_free_descs - free irq descriptors
782  * @from:	Start of descriptor range
799  * __irq_alloc_descs - allocate and initialize a range of irq descriptors
800  * @irq:	Allocate for specific irq number if irq >= 0
801  * @from:	Start the search from this irq number
803  * @node:	Preferred node on which the irq descriptor should be allocated
806  *		hints where the irq descriptors should be allocated and which
809  * Returns the first irq number or error code
811 int __ref __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,  in __irq_alloc_descs()  argument
814 	int start;  in __irq_alloc_descs()  local
817 		return -EINVAL;  in __irq_alloc_descs()
819 	if (irq >= 0) {  in __irq_alloc_descs()
820 		if (from > irq)  in __irq_alloc_descs()
821 			return -EINVAL;  in __irq_alloc_descs()
822 		from = irq;  in __irq_alloc_descs()
834 	start = irq_find_free_area(from, cnt);  in __irq_alloc_descs()
835 	if (irq >=0 && start != irq)  in __irq_alloc_descs()
836 		return -EEXIST;  in __irq_alloc_descs()
838 	if (start + cnt > nr_irqs) {  in __irq_alloc_descs()
839 		if (!irq_expand_nr_irqs(start + cnt))  in __irq_alloc_descs()
840 			return -ENOMEM;  in __irq_alloc_descs()
842 	return alloc_descs(start, cnt, node, affinity, owner);  in __irq_alloc_descs()
847  * irq_get_next_irq - get next allocated irq number
848  * @offset:	where to start the search
850  * Returns next irq number after offset or nr_irqs if none is found.
857 struct irq_desc *__irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,  in __irq_get_desc_lock()  argument
862 	desc = irq_to_desc(irq);  in __irq_get_desc_lock()
876 	raw_spin_lock_irqsave(&desc->lock, *flags);  in __irq_get_desc_lock()
882 	__releases(&desc->lock)  in __irq_put_desc_unlock()
884 	raw_spin_unlock_irqrestore(&desc->lock, flags);  in __irq_put_desc_unlock()
889 int irq_set_percpu_devid_partition(unsigned int irq,  in irq_set_percpu_devid_partition()  argument
892 	struct irq_desc *desc = irq_to_desc(irq);  in irq_set_percpu_devid_partition()
894 	if (!desc || desc->percpu_enabled)  in irq_set_percpu_devid_partition()
895 		return -EINVAL;  in irq_set_percpu_devid_partition()
897 	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);  in irq_set_percpu_devid_partition()
899 	if (!desc->percpu_enabled)  in irq_set_percpu_devid_partition()
900 		return -ENOMEM;  in irq_set_percpu_devid_partition()
902 	desc->percpu_affinity = affinity ? : cpu_possible_mask;  in irq_set_percpu_devid_partition()
904 	irq_set_percpu_devid_flags(irq);  in irq_set_percpu_devid_partition()
908 int irq_set_percpu_devid(unsigned int irq)  in irq_set_percpu_devid()  argument
910 	return irq_set_percpu_devid_partition(irq, NULL);  in irq_set_percpu_devid()
913 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)  in irq_get_percpu_devid_partition()  argument
915 	struct irq_desc *desc = irq_to_desc(irq);  in irq_get_percpu_devid_partition()
917 	if (!desc || !desc->percpu_enabled)  in irq_get_percpu_devid_partition()
918 		return -EINVAL;  in irq_get_percpu_devid_partition()
921 		cpumask_copy(affinity, desc->percpu_affinity);  in irq_get_percpu_devid_partition()
927 void kstat_incr_irq_this_cpu(unsigned int irq)  in kstat_incr_irq_this_cpu()  argument
929 	kstat_incr_irqs_this_cpu(irq_to_desc(irq));  in kstat_incr_irq_this_cpu()
933  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
934  * @irq:	The interrupt number
938  * @irq. The caller must ensure that the interrupt is not removed
941 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)  in kstat_irqs_cpu()  argument
943 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_irqs_cpu()
945 	return desc && desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0;  in kstat_irqs_cpu()
956 		return data_race(desc->tot_count);  in kstat_irqs_desc()
959 		sum += data_race(per_cpu(desc->kstat_irqs->cnt, cpu));  in kstat_irqs_desc()
963 static unsigned int kstat_irqs(unsigned int irq)  in kstat_irqs()  argument
965 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_irqs()
967 	if (!desc || !desc->kstat_irqs)  in kstat_irqs()
977 	unsigned int irq;  in kstat_snapshot_irqs()  local
979 	for_each_irq_desc(irq, desc) {  in kstat_snapshot_irqs()
980 		if (!desc->kstat_irqs)  in kstat_snapshot_irqs()
982 		this_cpu_write(desc->kstat_irqs->ref, this_cpu_read(desc->kstat_irqs->cnt));  in kstat_snapshot_irqs()
986 unsigned int kstat_get_irq_since_snapshot(unsigned int irq)  in kstat_get_irq_since_snapshot()  argument
988 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_get_irq_since_snapshot()
990 	if (!desc || !desc->kstat_irqs)  in kstat_get_irq_since_snapshot()
992 	return this_cpu_read(desc->kstat_irqs->cnt) - this_cpu_read(desc->kstat_irqs->ref);  in kstat_get_irq_since_snapshot()
998  * kstat_irqs_usr - Get the statistics for an interrupt from thread context
999  * @irq:	The interrupt number
1001  * Returns the sum of interrupt counts on all cpus since boot for @irq.
1007 unsigned int kstat_irqs_usr(unsigned int irq)  in kstat_irqs_usr()  argument
1012 	sum = kstat_irqs(irq);  in kstat_irqs_usr()
1018 void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,  in __irq_set_lockdep_class()  argument
1021 	struct irq_desc *desc = irq_to_desc(irq);  in __irq_set_lockdep_class()
1024 		lockdep_set_class(&desc->lock, lock_class);  in __irq_set_lockdep_class()
1025 		lockdep_set_class(&desc->request_mutex, request_class);  in __irq_set_lockdep_class()