55 static int alloc_masks(struct irq_desc *desc, int node)  in alloc_masks()  argument
57 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,  in alloc_masks()
62 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,  in alloc_masks()
64 		free_cpumask_var(desc->irq_common_data.affinity);  in alloc_masks()
70 	if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {  in alloc_masks()
72 		free_cpumask_var(desc->irq_common_data.effective_affinity);  in alloc_masks()
74 		free_cpumask_var(desc->irq_common_data.affinity);  in alloc_masks()
81 static void desc_smp_init(struct irq_desc *desc, int node,  in desc_smp_init()  argument
86 	cpumask_copy(desc->irq_common_data.affinity, affinity);  in desc_smp_init()
89 	cpumask_clear(desc->pending_mask);  in desc_smp_init()
92 	desc->irq_common_data.node = node;  in desc_smp_init()
96 static void free_masks(struct irq_desc *desc)  in free_masks()  argument
99 	free_cpumask_var(desc->pending_mask);  in free_masks()
101 	free_cpumask_var(desc->irq_common_data.affinity);  in free_masks()
103 	free_cpumask_var(desc->irq_common_data.effective_affinity);  in free_masks()
109 alloc_masks(struct irq_desc *desc, int node) { return 0; }  in alloc_masks()  argument
111 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }  in desc_smp_init()  argument
112 static inline void free_masks(struct irq_desc *desc) { }  in free_masks()  argument
115 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,  in desc_set_defaults()  argument
120 	desc->irq_common_data.handler_data = NULL;  in desc_set_defaults()
121 	desc->irq_common_data.msi_desc = NULL;  in desc_set_defaults()
123 	desc->irq_data.common = &desc->irq_common_data;  in desc_set_defaults()
124 	desc->irq_data.irq = irq;  in desc_set_defaults()
125 	desc->irq_data.chip = &no_irq_chip;  in desc_set_defaults()
126 	desc->irq_data.chip_data = NULL;  in desc_set_defaults()
127 	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);  in desc_set_defaults()
128 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);  in desc_set_defaults()
129 	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);  in desc_set_defaults()
130 	desc->handle_irq = handle_bad_irq;  in desc_set_defaults()
131 	desc->depth = 1;  in desc_set_defaults()
132 	desc->irq_count = 0;  in desc_set_defaults()
133 	desc->irqs_unhandled = 0;  in desc_set_defaults()
134 	desc->tot_count = 0;  in desc_set_defaults()
135 	desc->name = NULL;  in desc_set_defaults()
136 	desc->owner = owner;  in desc_set_defaults()
138 		*per_cpu_ptr(desc->kstat_irqs, cpu) = (struct irqstat) { };  in desc_set_defaults()
139 	desc_smp_init(desc, node, affinity);  in desc_set_defaults()
186 	struct irq_desc *desc;  in irq_find_at_or_after()  local
189 	desc = mt_find(&sparse_irqs, &index, nr_irqs);  in irq_find_at_or_after()
191 	return desc ? irq_desc_get_irq(desc) : nr_irqs;  in irq_find_at_or_after()
194 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)  in irq_insert_desc()  argument
197 	WARN_ON(mas_store_gfp(&mas, desc, GFP_KERNEL) != 0);  in irq_insert_desc()
210 static int init_desc(struct irq_desc *desc, int irq, int node,  in init_desc()  argument
215 	desc->kstat_irqs = alloc_percpu(struct irqstat);  in init_desc()
216 	if (!desc->kstat_irqs)  in init_desc()
219 	if (alloc_masks(desc, node)) {  in init_desc()
220 		free_percpu(desc->kstat_irqs);  in init_desc()
224 	raw_spin_lock_init(&desc->lock);  in init_desc()
225 	lockdep_set_class(&desc->lock, &irq_desc_lock_class);  in init_desc()
226 	mutex_init(&desc->request_mutex);  in init_desc()
227 	init_waitqueue_head(&desc->wait_for_threads);  in init_desc()
228 	desc_set_defaults(irq, desc, node, affinity, owner);  in init_desc()
229 	irqd_set(&desc->irq_data, flags);  in init_desc()
230 	irq_resend_init(desc);  in init_desc()
232 	kobject_init(&desc->kobj, &irq_kobj_type);  in init_desc()
233 	init_rcu_head(&desc->rcu);  in init_desc()
251 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in per_cpu_count_show()  local
257 		unsigned int c = irq_desc_kstat_cpu(desc, cpu);  in per_cpu_count_show()
270 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in chip_name_show()  local
272 	guard(raw_spinlock_irq)(&desc->lock);  in chip_name_show()
273 	if (desc->irq_data.chip && desc->irq_data.chip->name)  in chip_name_show()
274 		return sysfs_emit(buf, "%s\n", desc->irq_data.chip->name);  in chip_name_show()
281 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in hwirq_show()  local
283 	guard(raw_spinlock_irq)(&desc->lock);  in hwirq_show()
284 	if (desc->irq_data.domain)  in hwirq_show()
285 		return sysfs_emit(buf, "%lu\n", desc->irq_data.hwirq);  in hwirq_show()
292 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in type_show()  local
294 	guard(raw_spinlock_irq)(&desc->lock);  in type_show()
295 	return sysfs_emit(buf, "%s\n", irqd_is_level_type(&desc->irq_data) ? "level" : "edge");  in type_show()
302 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in wakeup_show()  local
304 	guard(raw_spinlock_irq)(&desc->lock);  in wakeup_show()
305 	return sysfs_emit(buf, "%s\n", str_enabled_disabled(irqd_is_wakeup_set(&desc->irq_data)));  in wakeup_show()
311 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in name_show()  local
313 	guard(raw_spinlock_irq)(&desc->lock);  in name_show()
314 	if (desc->name)  in name_show()
315 		return sysfs_emit(buf, "%s\n", desc->name);  in name_show()
322 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in actions_show()  local
327 	scoped_guard(raw_spinlock_irq, &desc->lock) {  in actions_show()
328 		for_each_action_of_desc(desc, action) {  in actions_show()
358 static void irq_sysfs_add(int irq, struct irq_desc *desc)  in irq_sysfs_add()  argument
366 		if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))  in irq_sysfs_add()
369 			desc->istate |= IRQS_SYSFS;  in irq_sysfs_add()
373 static void irq_sysfs_del(struct irq_desc *desc)  in irq_sysfs_del()  argument
381 	if (desc->istate & IRQS_SYSFS)  in irq_sysfs_del()
382 		kobject_del(&desc->kobj);  in irq_sysfs_del()
387 	struct irq_desc *desc;  in irq_sysfs_init()  local
397 	for_each_irq_desc(irq, desc)  in irq_sysfs_init()
398 		irq_sysfs_add(irq, desc);  in irq_sysfs_init()
409 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}  in irq_sysfs_add()  argument
410 static void irq_sysfs_del(struct irq_desc *desc) {}  in irq_sysfs_del()  argument
436 	struct irq_desc *desc;  in alloc_desc()  local
439 	desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node);  in alloc_desc()
440 	if (!desc)  in alloc_desc()
443 	ret = init_desc(desc, irq, node, flags, affinity, owner);  in alloc_desc()
445 		kfree(desc);  in alloc_desc()
449 	return desc;  in alloc_desc()
454 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);  in irq_kobj_release()  local
456 	free_masks(desc);  in irq_kobj_release()
457 	free_percpu(desc->kstat_irqs);  in irq_kobj_release()
458 	kfree(desc);  in irq_kobj_release()
463 	struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);  in delayed_free_desc()  local
465 	kobject_put(&desc->kobj);  in delayed_free_desc()
470 	struct irq_desc *desc = irq_to_desc(irq);  in free_desc()  local
472 	irq_remove_debugfs_entry(desc);  in free_desc()
473 	unregister_irq_proc(irq, desc);  in free_desc()
484 	irq_sysfs_del(desc);  in free_desc()
493 	call_rcu(&desc->rcu, delayed_free_desc);  in free_desc()
500 	struct irq_desc *desc;  in alloc_descs()  local
526 		desc = alloc_desc(start + i, node, flags, mask, owner);  in alloc_descs()
527 		if (!desc)  in alloc_descs()
529 		irq_insert_desc(start + i, desc);  in alloc_descs()
530 		irq_sysfs_add(start + i, desc);  in alloc_descs()
531 		irq_add_debugfs_entry(start + i, desc);  in alloc_descs()
552 	struct irq_desc *desc;  in early_irq_init()  local
571 		desc = alloc_desc(i, node, 0, NULL, NULL);  in early_irq_init()
572 		irq_insert_desc(i, desc);  in early_irq_init()
623 	struct irq_desc *desc = irq_to_desc(irq);  in free_desc()  local
625 	scoped_guard(raw_spinlock_irqsave, &desc->lock)  in free_desc()
626 		desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);  in free_desc()
637 		struct irq_desc *desc = irq_to_desc(start + i);  in alloc_descs()  local
639 		desc->owner = owner;  in alloc_descs()
640 		irq_insert_desc(start + i, desc);  in alloc_descs()
658 int handle_irq_desc(struct irq_desc *desc)  in handle_irq_desc()  argument
662 	if (!desc)  in handle_irq_desc()
665 	data = irq_desc_get_irq_data(desc);  in handle_irq_desc()
669 	generic_handle_irq_desc(desc);  in handle_irq_desc()
853 	struct irq_desc *desc;  in __irq_get_desc_lock()  local
855 	desc = irq_to_desc(irq);  in __irq_get_desc_lock()
856 	if (!desc)  in __irq_get_desc_lock()
860 		if ((check & _IRQ_DESC_PERCPU) && !irq_settings_is_per_cpu_devid(desc))  in __irq_get_desc_lock()
863 		if (!(check & _IRQ_DESC_PERCPU) && irq_settings_is_per_cpu_devid(desc))  in __irq_get_desc_lock()
868 		chip_bus_lock(desc);  in __irq_get_desc_lock()
869 	raw_spin_lock_irqsave(&desc->lock, *flags);  in __irq_get_desc_lock()
871 	return desc;  in __irq_get_desc_lock()
874 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)  in __irq_put_desc_unlock()  argument
875 	__releases(&desc->lock)  in __irq_put_desc_unlock()
877 	raw_spin_unlock_irqrestore(&desc->lock, flags);  in __irq_put_desc_unlock()
879 		chip_bus_sync_unlock(desc);  in __irq_put_desc_unlock()
885 	struct irq_desc *desc = irq_to_desc(irq);  in irq_set_percpu_devid_partition()  local
887 	if (!desc || desc->percpu_enabled)  in irq_set_percpu_devid_partition()
890 	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);  in irq_set_percpu_devid_partition()
892 	if (!desc->percpu_enabled)  in irq_set_percpu_devid_partition()
895 	desc->percpu_affinity = affinity ? : cpu_possible_mask;  in irq_set_percpu_devid_partition()
908 	struct irq_desc *desc = irq_to_desc(irq);  in irq_get_percpu_devid_partition()  local
910 	if (!desc || !desc->percpu_enabled)  in irq_get_percpu_devid_partition()
914 		cpumask_copy(affinity, desc->percpu_affinity);  in irq_get_percpu_devid_partition()
936 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_irqs_cpu()  local
938 	return desc && desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0;  in kstat_irqs_cpu()
941 static unsigned int kstat_irqs_desc(struct irq_desc *desc, const struct cpumask *cpumask)  in kstat_irqs_desc()  argument
946 	if (!irq_settings_is_per_cpu_devid(desc) &&  in kstat_irqs_desc()
947 	    !irq_settings_is_per_cpu(desc) &&  in kstat_irqs_desc()
948 	    !irq_is_nmi(desc))  in kstat_irqs_desc()
949 		return data_race(desc->tot_count);  in kstat_irqs_desc()
952 		sum += data_race(per_cpu(desc->kstat_irqs->cnt, cpu));  in kstat_irqs_desc()
958 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_irqs()  local
960 	if (!desc || !desc->kstat_irqs)  in kstat_irqs()
962 	return kstat_irqs_desc(desc, cpu_possible_mask);  in kstat_irqs()
969 	struct irq_desc *desc;  in kstat_snapshot_irqs()  local
972 	for_each_irq_desc(irq, desc) {  in kstat_snapshot_irqs()
973 		if (!desc->kstat_irqs)  in kstat_snapshot_irqs()
975 		this_cpu_write(desc->kstat_irqs->ref, this_cpu_read(desc->kstat_irqs->cnt));  in kstat_snapshot_irqs()
981 	struct irq_desc *desc = irq_to_desc(irq);  in kstat_get_irq_since_snapshot()  local
983 	if (!desc || !desc->kstat_irqs)  in kstat_get_irq_since_snapshot()
985 	return this_cpu_read(desc->kstat_irqs->cnt) - this_cpu_read(desc->kstat_irqs->ref);  in kstat_get_irq_since_snapshot()
1014 	struct irq_desc *desc = irq_to_desc(irq);  in __irq_set_lockdep_class()  local
1016 	if (desc) {  in __irq_set_lockdep_class()
1017 		lockdep_set_class(&desc->lock, lock_class);  in __irq_set_lockdep_class()
1018 		lockdep_set_class(&desc->request_mutex, request_class);  in __irq_set_lockdep_class()